Keystone backend preparation for domain-scoping

These changes lay the ground work for the implmentation of
domain-scoping, but are benign in that they don't change the token.
They include making domain_id a first-class attribute in the user
and project entity (i.e. move it out of the 'extra' attribute),
filling in domain grant and project support for the kvs backend and
fixing a series of issues in the mirgation to make it work for both
MySQL, Postgresql and sqlite.

A further, separate, commit will actually provide the code to
update the actual tokens once the v3 token support has been added.

blueprint domain-scoping
blueprint default-domain

Change-Id: I55ab7947a6a1efbab003bd234856bd3805bb4a63

keystone/common/controller.py

@@ -10,6 +10,7 @@ from keystone import exception
 
 LOG = logging.getLogger(__name__)
 CONF = config.CONF
+DEFAULT_DOMAIN_ID = CONF.identity.default_domain_id
 
 
 def protected(f):
@@ -68,6 +69,21 @@ class V2Controller(wsgi.Application):
             msg = '%s field is required and cannot be empty' % attr
             raise exception.ValidationError(message=msg)
 
+    def _normalize_domain_id(self, context, ref):
+        """Fill in domain_id since v2 calls are not domain-aware.
+
+        This will overwrite any domain_id that was inadvertently
+        specified in the v2 call.
+
+        """
+        ref['domain_id'] = DEFAULT_DOMAIN_ID
+        return ref
+
+    def _filter_domain_id(self, ref):
+        """Remove domain_id since v2 calls are not domain-aware."""
+        ref.pop('domain_id', None)
+        return ref
+
 
 class V3Controller(V2Controller):
     """Base controller class for Identity API v3.
@@ -148,3 +164,37 @@ class V3Controller(V2Controller):
             value = context['query_string'][attr]
             return [r for r in refs if r[attr] == value]
         return refs
+
+    def _normalize_domain_id(self, context, ref):
+        """Fill in domain_id if not specified in a v3 call."""
+
+        if 'domain_id' not in ref:
+            if context['is_admin']:
+                ref['domain_id'] = DEFAULT_DOMAIN_ID
+            else:
+                # Fish the domain_id out of the token
+                #
+                # We could make this more efficient by loading the domain_id
+                # into the context in the wrapper function above (since
+                # this version of normalize_domain will only be called inside
+                # a v3 protected call).  However, given that we only use this
+                # for creating entities, this optimization is probably not
+                # worth the duplication of state
+                try:
+                    token_ref = self.token_api.get_token(
+                        context=context, token_id=context['token_id'])
+                except exception.TokenNotFound:
+                    LOG.warning(_('Invalid token in normalize_domain_id'))
+                    raise exception.Unauthorized()
+
+                if 'domain' in token_ref:
+                    ref['domain_id'] = token_ref['domain']['id']
+                else:
+                    # FIXME(henry-nash) Revisit this once v3 token scoping
+                    # across domains has been hashed out
+                    ref['domain_id'] = DEFAULT_DOMAIN_ID
+        return ref
+
+    def _filter_domain_id(self, ref):
+        """Override v2 filter to let domain_id out for v3 calls."""
+        return ref

keystone/common/models.py

@@ -87,6 +87,7 @@ class User(Model):
     Required keys:
         id
         name
+        domain_id
 
     Optional keys:
         password
@@ -95,7 +96,7 @@ class User(Model):
         enabled (bool, default True)
     """
 
-    required_keys = ('id', 'name')
+    required_keys = ('id', 'name', 'domain_id')
     optional_keys = ('password', 'description', 'email', 'enabled')
 
 
@@ -105,15 +106,16 @@ class Group(Model):
     Required keys:
         id
         name
+        domain_id
 
     Optional keys:
-        domain_id
+
         description
 
     """
 
-    required_keys = ('id', 'name')
+    required_keys = ('id', 'name', 'domain_id')
-    optional_keys = ('domain_id', 'description')
+    optional_keys = ('description')
 
 
 class Project(Model):
@@ -122,6 +124,7 @@ class Project(Model):
     Required keys:
         id
         name
+        domain_id
 
     Optional Keys:
         description
@@ -129,7 +132,7 @@ class Project(Model):
 
     """
 
-    required_keys = ('id', 'name')
+    required_keys = ('id', 'name', 'domain_id')
     optional_keys = ('description', 'enabled')
 
 

keystone/common/sql/legacy.py

@@ -22,9 +22,12 @@ from sqlalchemy import exc
 from keystone.common import logging
 from keystone.contrib.ec2.backends import sql as ec2_sql
 from keystone.identity.backends import sql as identity_sql
+from keystone import config
 
 
 LOG = logging.getLogger(__name__)
+CONF = config.CONF
+DEFAULT_DOMAIN_ID = CONF.identity.default_domain_id
 
 
 def export_db(db):
@@ -103,7 +106,8 @@ class LegacyMigration(object):
             # map
             new_dict = {'description': x.get('desc', ''),
                         'id': x.get('uid', x.get('id')),
-                        'enabled': x.get('enabled', True)}
+                        'enabled': x.get('enabled', True),
+                        'domain_id': x.get('domain_id', DEFAULT_DOMAIN_ID)}
             new_dict['name'] = x.get('name', new_dict.get('id'))
             # track internal ids
             self._project_map[x.get('id')] = new_dict['id']
@@ -117,7 +121,8 @@ class LegacyMigration(object):
             new_dict = {'email': x.get('email', ''),
                         'password': x.get('password', None),
                         'id': x.get('uid', x.get('id')),
-                        'enabled': x.get('enabled', True)}
+                        'enabled': x.get('enabled', True),
+                        'domain_id': x.get('domain_id', DEFAULT_DOMAIN_ID)}
             if x.get('tenant_id'):
                 new_dict['tenant_id'] = self._project_map.get(x['tenant_id'])
             new_dict['name'] = x.get('name', new_dict.get('id'))

keystone/common/sql/migrate_repo/versions/008_create_default_domain.py

@@ -22,7 +22,7 @@ from keystone import config
 
 
 CONF = config.CONF
-DEFAULT_DOMAIN_ID = CONF['identity']['default_domain_id']
+DEFAULT_DOMAIN_ID = CONF.identity.default_domain_id
 
 
 def upgrade(migrate_engine):

keystone/common/sql/migrate_repo/versions/010_normalize_identity_migration.py

@@ -32,26 +32,21 @@ def is_enabled(enabled):
     return bool(enabled)
 
 
-def downgrade_user_table(meta, migrate_engine):
+def downgrade_user_table(meta, migrate_engine, session):
     user_table = Table('user', meta, autoload=True)
-    maker = sessionmaker(bind=migrate_engine)
-    session = maker()
     for user in session.query(user_table).all():
         extra = json.loads(user.extra)
         extra['password'] = user.password
         extra['enabled'] = '%r' % user.enabled
-        values = {'extra':  json.dumps(extra)}
+        values = {'extra': json.dumps(extra)}
         update = user_table.update().\
             where(user_table.c.id == user.id).\
             values(values)
         migrate_engine.execute(update)
-    session.commit()
 
 
-def downgrade_tenant_table(meta, migrate_engine):
+def downgrade_tenant_table(meta, migrate_engine, session):
     tenant_table = Table('tenant', meta, autoload=True)
-    maker = sessionmaker(bind=migrate_engine)
-    session = maker()
     for tenant in session.query(tenant_table).all():
         extra = json.loads(tenant.extra)
         extra['description'] = tenant.description
@@ -61,13 +56,10 @@ def downgrade_tenant_table(meta, migrate_engine):
             where(tenant_table.c.id == tenant.id).\
             values(values)
         migrate_engine.execute(update)
-    session.commit()
 
 
-def upgrade_user_table(meta, migrate_engine):
+def upgrade_user_table(meta, migrate_engine, session):
     user_table = Table('user', meta, autoload=True)
-    maker = sessionmaker(bind=migrate_engine)
-    session = maker()
     for user in session.query(user_table).all():
         extra = json.loads(user.extra)
         values = {'password': extra.pop('password', None),
@@ -77,14 +69,10 @@ def upgrade_user_table(meta, migrate_engine):
             where(user_table.c.id == user.id).\
             values(values)
         migrate_engine.execute(update)
-    session.commit()
 
 
-def upgrade_tenant_table(meta, migrate_engine):
+def upgrade_tenant_table(meta, migrate_engine, session):
     tenant_table = Table('tenant', meta, autoload=True)
-
-    maker = sessionmaker(bind=migrate_engine)
-    session = maker()
     for tenant in session.query(tenant_table):
         extra = json.loads(tenant.extra)
         values = {'description': extra.pop('description', None),
@@ -94,18 +82,21 @@ def upgrade_tenant_table(meta, migrate_engine):
             where(tenant_table.c.id == tenant.id).\
             values(values)
         migrate_engine.execute(update)
-    session.commit()
 
 
 def upgrade(migrate_engine):
     meta = MetaData()
     meta.bind = migrate_engine
-    upgrade_user_table(meta, migrate_engine)
+    session = sessionmaker(bind=migrate_engine)()
-    upgrade_tenant_table(meta, migrate_engine)
+    upgrade_user_table(meta, migrate_engine, session)
+    upgrade_tenant_table(meta, migrate_engine, session)
+    session.commit()
 
 
 def downgrade(migrate_engine):
     meta = MetaData()
     meta.bind = migrate_engine
-    downgrade_user_table(meta, migrate_engine)
+    session = sessionmaker(bind=migrate_engine)()
-    downgrade_tenant_table(meta, migrate_engine)
+    downgrade_user_table(meta, migrate_engine, session)
+    downgrade_tenant_table(meta, migrate_engine, session)
+    session.commit()

keystone/common/sql/migrate_repo/versions/012_populate_endpoint_type.py

@@ -48,13 +48,10 @@ def upgrade(migrate_engine):
                 'url': urls[interface],
                 'extra': json.dumps(extra),
             }
-            session.execute(
+            insert = new_table.insert().values(endpoint)
-                'INSERT INTO `%s` (%s) VALUES (%s)' % (
+            migrate_engine.execute(insert)
-                    new_table.name,
-                    ', '.join('%s' % k for k in endpoint.keys()),
-                    ', '.join([':%s' % k for k in endpoint.keys()])),
-                endpoint)
     session.commit()
+    session.close()
 
 
 def downgrade(migrate_engine):
@@ -67,31 +64,31 @@ def downgrade(migrate_engine):
 
     session = orm.sessionmaker(bind=migrate_engine)()
     for ref in session.query(new_table).all():
-        extra = json.loads(ref.extra)
+        q = session.query(legacy_table)
-        extra['%surl' % ref.interface] = ref.url
+        q = q.filter_by(id=ref.legacy_endpoint_id)
-        endpoint = {
+        legacy_ref = q.first()
-            'id': ref.legacy_endpoint_id,
+        if legacy_ref:
-            'region': ref.region,
+            # We already have one, so just update the extra
-            'service_id': ref.service_id,
+            # attribute with the urls.
-            'extra': json.dumps(extra),
-        }
-
-        try:
-            session.execute(
-                'INSERT INTO `%s` (%s) VALUES (%s)' % (
-                    legacy_table.name,
-                    ', '.join('%s' % k for k in endpoint.keys()),
-                    ', '.join([':%s' % k for k in endpoint.keys()])),
-                endpoint)
-        except sql.exc.IntegrityError:
-            q = session.query(legacy_table)
-            q = q.filter_by(id=ref.legacy_endpoint_id)
-            legacy_ref = q.one()
             extra = json.loads(legacy_ref.extra)
             extra['%surl' % ref.interface] = ref.url
-
+            values = {'extra': json.dumps(extra)}
-            session.execute(
+            update = legacy_table.update().\
-                'UPDATE `%s` SET extra=:extra WHERE id=:id' % (
+                where(legacy_table.c.id == legacy_ref.id).\
-                    legacy_table.name),
+                values(values)
-                {'extra': json.dumps(extra), 'id': legacy_ref.id})
+            migrate_engine.execute(update)
-    session.commit()
+        else:
+            # This is the first one of this legacy ID, so
+            # we can insert instead.
+            extra = json.loads(ref.extra)
+            extra['%surl' % ref.interface] = ref.url
+            endpoint = {
+                'id': ref.legacy_endpoint_id,
+                'region': ref.region,
+                'service_id': ref.service_id,
+                'extra': json.dumps(extra),
+            }
+            insert = legacy_table.insert().values(endpoint)
+            migrate_engine.execute(insert)
+        session.commit()
+    session.close()

keystone/common/sql/migrate_repo/versions/014_add_group_tables.py

@@ -26,7 +26,8 @@ def upgrade(migrate_engine):
         'group',
         meta,
         sql.Column('id', sql.String(64), primary_key=True),
-        sql.Column('domain_id', sql.String(64), sql.ForeignKey('domain.id')),
+        sql.Column('domain_id', sql.String(64), sql.ForeignKey('domain.id'),
+                   nullable=False),
         sql.Column('name', sql.String(64), unique=True, nullable=False),
         sql.Column('description', sql.Text()),
         sql.Column('extra', sql.Text()))

keystone/common/sql/migrate_repo/versions/015_tenant_to_project.py

@@ -11,7 +11,6 @@ def upgrade(migrate_engine):
 
 
 def downgrade(migrate_engine):
-    """Replace API-version specific endpoint tables with one based on v2."""
     meta = sql.MetaData()
     meta.bind = migrate_engine
     upgrade_table = sql.Table('project', meta, autoload=True)

keystone/common/sql/migrate_repo/versions/016_normalize_domain_ids.py

@@ -0,0 +1,414 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2012 OpenStack LLC
+# Copyright 2013 IBM
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+Normalize for domain_id, i.e. ensure User and Project entities have the
+domain_id as a first class attribute.
+
+Both User and Project (as well as Group) entities are owned by a
+domain, which is implemented as each having a domain_id foreign key
+in their sql representation that points back to the respective
+domain in the domain table.  This domain_id attribute should also
+be required (i.e. not nullable)
+
+Adding a non_nullable foreign key attribute to a table with existing
+data causes a few problems since not all DB engines support the
+ability to either control the triggering of integrity constraints
+or the ability to modify columns after they are created.
+
+To get round the above inconsistencies, two versions of the
+upgrade/downgrade functions are supplied, one for those engines
+that support dropping columns, and one for those that don't.  For
+the latter we are forced to do table copy AND control the triggering
+of integrity constraints.
+"""
+
+import sqlalchemy as sql
+from sqlalchemy.orm import sessionmaker
+from keystone import config
+
+
+CONF = config.CONF
+DEFAULT_DOMAIN_ID = CONF.identity.default_domain_id
+
+
+def _disable_foreign_constraints(session, migrate_engine):
+    if migrate_engine.name == 'mysql':
+        session.execute('SET foreign_key_checks = 0;')
+
+
+def _enable_foreign_constraints(session, migrate_engine):
+    if migrate_engine.name == 'mysql':
+        session.execute('SET foreign_key_checks = 1;')
+
+
+def upgrade_user_table_with_copy(meta, migrate_engine, session):
+    # We want to add the domain_id attribute to the user table.  Since
+    # it is non nullable and the table may have data, easiest way is
+    # a table copy. Further, in order to keep foreign key constraints
+    # pointing at the right table, we need to be able and do a table
+    # DROP then CREATE, rather than ALTERing the name of the table.
+
+    # First make a copy of the user table
+    temp_user_table = sql.Table(
+        'temp_user',
+        meta,
+        sql.Column('id', sql.String(64), primary_key=True),
+        sql.Column('name', sql.String(64), unique=True, nullable=False),
+        sql.Column('extra', sql.Text()),
+        sql.Column("password", sql.String(128)),
+        sql.Column("enabled", sql.Boolean, default=True))
+    temp_user_table.create(migrate_engine, checkfirst=True)
+
+    user_table = sql.Table('user', meta, autoload=True)
+    for user in session.query(user_table):
+        session.execute("insert into temp_user (id, name, extra, "
+                        "password, enabled) "
+                        "values ( :id, :name, :extra, "
+                        ":password, :enabled);",
+                        {'id': user.id,
+                         'name': user.name,
+                         'extra': user.extra,
+                         'password': user.password,
+                         'enabled': user.enabled})
+
+    # Now switch off constraints while we drop and then re-create the
+    # user table, with the additional domain_id column
+    _disable_foreign_constraints(session, migrate_engine)
+    session.execute('drop table user;')
+    # Need to create a new metadata stream since we are going to load a
+    # different version of the user table
+    meta2 = sql.MetaData()
+    meta2.bind = migrate_engine
+    domain_table = sql.Table('domain', meta2, autoload=True)
+    user_table = sql.Table(
+        'user',
+        meta2,
+        sql.Column('id', sql.String(64), primary_key=True),
+        sql.Column('name', sql.String(64), unique=True, nullable=False),
+        sql.Column('extra', sql.Text()),
+        sql.Column("password", sql.String(128)),
+        sql.Column("enabled", sql.Boolean, default=True),
+        sql.Column('domain_id', sql.String(64), sql.ForeignKey('domain.id'),
+                   nullable=False))
+    user_table.create(migrate_engine, checkfirst=True)
+
+    # Finally copy in the data from our temp table and then clean
+    # up by deleting our temp table
+    for user in session.query(temp_user_table):
+        session.execute("insert into user (id, name, extra, "
+                        "password, enabled, domain_id) "
+                        "values ( :id, :name, :extra, "
+                        ":password, :enabled, :domain_id);",
+                        {'id': user.id,
+                         'name': user.name,
+                         'extra': user.extra,
+                         'password': user.password,
+                         'enabled': user.enabled,
+                         'domain_id': DEFAULT_DOMAIN_ID})
+    _enable_foreign_constraints(session, migrate_engine)
+    session.execute("drop table temp_user;")
+
+
+def upgrade_project_table_with_copy(meta, migrate_engine, session):
+    # We want to add the domain_id attribute to the project table.  Since
+    # it is non nullable and the table may have data, easiest way is
+    # a table copy. Further, in order to keep foreign key constraints
+    # pointing at the right table, we need to be able and do a table
+    # DROP then CREATE, rather than ALTERing the name of the table.
+
+    # Fist make a copy of the project table
+    temp_project_table = sql.Table(
+        'temp_project',
+        meta,
+        sql.Column('id', sql.String(64), primary_key=True),
+        sql.Column('name', sql.String(64), unique=True, nullable=False),
+        sql.Column('extra', sql.Text()),
+        sql.Column("description", sql.Text()),
+        sql.Column("enabled", sql.Boolean, default=True))
+    temp_project_table.create(migrate_engine, checkfirst=True)
+
+    project_table = sql.Table('project', meta, autoload=True)
+    for project in session.query(project_table):
+        session.execute("insert into temp_project (id, name, extra, "
+                        "description, enabled) "
+                        "values ( :id, :name, :extra, "
+                        ":description, :enabled);",
+                        {'id': project.id,
+                         'name': project.name,
+                         'extra': project.extra,
+                         'description': project.description,
+                         'enabled': project.enabled})
+
+    # Now switch off constraints while we drop and then re-create the
+    # project table, with the additional domain_id column
+    _disable_foreign_constraints(session, migrate_engine)
+    session.execute("drop table project;")
+    # Need to create a new metadata stream since we are going to load a
+    # different version of the project table
+    meta2 = sql.MetaData()
+    meta2.bind = migrate_engine
+    domain_table = sql.Table('domain', meta2, autoload=True)
+    project_table = sql.Table(
+        'project',
+        meta2,
+        sql.Column('id', sql.String(64), primary_key=True),
+        sql.Column('name', sql.String(64), unique=True, nullable=False),
+        sql.Column('extra', sql.Text()),
+        sql.Column('description', sql.Text()),
+        sql.Column("enabled", sql.Boolean, default=True),
+        sql.Column('domain_id', sql.String(64), sql.ForeignKey('domain.id'),
+                   nullable=False))
+    project_table.create(migrate_engine, checkfirst=True)
+
+    # Finally copy in the data from our temp table and then clean
+    # up by deleting our temp table
+    for project in session.query(temp_project_table):
+        session.execute("insert into project (id, name, extra, "
+                        "description, enabled, domain_id) "
+                        "values ( :id, :name, :extra, "
+                        ":description, :enabled, :domain_id);",
+                        {'id': project.id,
+                         'name': project.name,
+                         'extra': project.extra,
+                         'description': project.description,
+                         'enabled': project.enabled,
+                         'domain_id': DEFAULT_DOMAIN_ID})
+    _enable_foreign_constraints(session, migrate_engine)
+    session.execute("drop table temp_project;")
+
+
+def downgrade_user_table_with_copy(meta, migrate_engine, session):
+    # For engines that don't support dropping columns, we need to do this
+    # as a table copy.  Further, in order to keep foreign key constraints
+    # pointing at the right table, we need to be able and do a table
+    # DROP then CREATE, rather than ALTERing the name of the table.
+
+    # Fist make a copy of the user table
+    temp_user_table = sql.Table(
+        'temp_user',
+        meta,
+        sql.Column('id', sql.String(64), primary_key=True),
+        sql.Column('name', sql.String(64), unique=True, nullable=False),
+        # Temporary table, so no need to make it a foreign key
+        sql.Column('domain_id', sql.String(64), nullable=False),
+        sql.Column("password", sql.String(128)),
+        sql.Column("enabled", sql.Boolean, default=True),
+        sql.Column('extra', sql.Text()))
+    temp_user_table.create(migrate_engine, checkfirst=True)
+
+    user_table = sql.Table('user', meta, autoload=True)
+    for user in session.query(user_table):
+        session.execute("insert into temp_user (id, name, domain_id, "
+                        "password, enabled, extra) "
+                        "values ( :id, :name, :domain_id, "
+                        ":password, :enabled, :extra);",
+                        {'id': user.id,
+                         'name': user.name,
+                         'domain_id': user.domain_id,
+                         'password': user.password,
+                         'enabled': user.enabled,
+                         'extra': user.extra})
+
+    # Now switch off constraints while we drop and then re-create the
+    # user table, less the columns we wanted to drop
+    _disable_foreign_constraints(session, migrate_engine)
+    session.execute("drop table user;")
+    # Need to create a new metadata stream since we are going to load a
+    # different version of the user table
+    meta2 = sql.MetaData()
+    meta2.bind = migrate_engine
+    user_table = sql.Table(
+        'user',
+        meta2,
+        sql.Column('id', sql.String(64), primary_key=True),
+        sql.Column('name', sql.String(64), unique=True, nullable=False),
+        sql.Column('extra', sql.Text()),
+        sql.Column("password", sql.String(128)),
+        sql.Column("enabled", sql.Boolean, default=True))
+    user_table.create(migrate_engine, checkfirst=True)
+    _enable_foreign_constraints(session, migrate_engine)
+
+    # Finally copy in the data from our temp table and then clean
+    # up by deleting our temp table
+    for user in session.query(temp_user_table):
+        session.execute("insert into user (id, name, extra, "
+                        "password, enabled) "
+                        "values ( :id, :name, :extra, "
+                        ":password, :enabled);",
+                        {'id': user.id,
+                         'name': user.name,
+                         'extra': user.extra,
+                         'password': user.password,
+                         'enabled': user.enabled})
+    session.execute("drop table temp_user;")
+
+
+def downgrade_project_table_with_copy(meta, migrate_engine, session):
+    # For engines that don't support dropping columns, we need to do this
+    # as a table copy.  Further, in order to keep foreign key constraints
+    # pointing at the right table, we need to be able and do a table
+    # DROP then CREATE, rather than ALTERing the name of the table.
+
+    # Fist make a copy of the project table
+    temp_project_table = sql.Table(
+        'temp_project',
+        meta,
+        sql.Column('id', sql.String(64), primary_key=True),
+        sql.Column('name', sql.String(64), unique=True, nullable=False),
+        # Temporary table, so no need to make it a foreign key
+        sql.Column('domain_id', sql.String(64), nullable=False),
+        sql.Column('description', sql.Text()),
+        sql.Column("enabled", sql.Boolean, default=True),
+        sql.Column('extra', sql.Text()))
+    temp_project_table.create(migrate_engine, checkfirst=True)
+
+    project_table = sql.Table('project', meta, autoload=True)
+    for project in session.query(project_table):
+        session.execute("insert into temp_project (id, name, domain_id, "
+                        "description, enabled, extra) "
+                        "values ( :id, :name, :domain_id, "
+                        ":description, :enabled, :extra);",
+                        {'id': project.id,
+                         'name': project.name,
+                         'domain_id': project.domain_id,
+                         'description': project.description,
+                         'enabled': project.enabled,
+                         'extra': project.extra})
+
+    # Now switch off constraints while we drop and then re-create the
+    # project table, less the columns we wanted to drop
+    _disable_foreign_constraints(session, migrate_engine)
+    session.execute("drop table project;")
+    # Need to create a new metadata stream since we are going to load a
+    # different version of the project table
+    meta2 = sql.MetaData()
+    meta2.bind = migrate_engine
+    project_table = sql.Table(
+        'project',
+        meta2,
+        sql.Column('id', sql.String(64), primary_key=True),
+        sql.Column('name', sql.String(64), unique=True, nullable=False),
+        sql.Column('extra', sql.Text()),
+        sql.Column("description", sql.Text()),
+        sql.Column("enabled", sql.Boolean, default=True))
+    project_table.create(migrate_engine, checkfirst=True)
+    _enable_foreign_constraints(session, migrate_engine)
+
+    # Finally copy in the data from our temp table and then clean
+    # up by deleting our temp table
+    for project in session.query(temp_project_table):
+        session.execute("insert into project (id, name, extra, "
+                        "description, enabled) "
+                        "values ( :id, :name, :extra, "
+                        ":description, :enabled);",
+                        {'id': project.id,
+                         'name': project.name,
+                         'extra': project.extra,
+                         'description': project.description,
+                         'enabled': project.enabled})
+    session.execute("drop table temp_project;")
+
+
+def upgrade_user_table_with_col_create(meta, migrate_engine, session):
+    # Create the domain_id column.  We want this to be not nullable
+    # but also a foreign key.  We can't create this right off the
+    # bat since any existing rows would cause an Integrity Error.
+    # We therefore create it nullable, fill the column with the
+    # default data and then set it to non nullable.
+    domain_table = sql.Table('domain', meta, autoload=True)
+    user_table = sql.Table('user', meta, autoload=True)
+    user_table.create_column(
+        sql.Column('domain_id', sql.String(64),
+                   sql.ForeignKey('domain.id'), nullable=True))
+    for user in session.query(user_table).all():
+        values = {'domain_id': DEFAULT_DOMAIN_ID}
+        update = user_table.update().\
+            where(user_table.c.id == user.id).\
+            values(values)
+        migrate_engine.execute(update)
+    # Need to commit this or setting nullable to False will fail
+    session.commit()
+    user_table.columns.domain_id.alter(nullable=False)
+
+
+def upgrade_project_table_with_col_create(meta, migrate_engine, session):
+    # Create the domain_id column.  We want this to be not nullable
+    # but also a foreign key.  We can't create this right off the
+    # bat since any existing rows would cause an Integrity Error.
+    # We therefore create it nullable, fill the column with the
+    # default data and then set it to non nullable.
+    domain_table = sql.Table('domain', meta, autoload=True)
+    project_table = sql.Table('project', meta, autoload=True)
+    project_table.create_column(
+        sql.Column('domain_id', sql.String(64),
+                   sql.ForeignKey('domain.id'), nullable=True))
+    for project in session.query(project_table).all():
+        values = {'domain_id': DEFAULT_DOMAIN_ID}
+        update = project_table.update().\
+            where(project_table.c.id == project.id).\
+            values(values)
+        migrate_engine.execute(update)
+    # Need to commit this or setting nullable to False will fail
+    session.commit()
+    project_table.columns.domain_id.alter(nullable=False)
+
+
+def downgrade_user_table_with_col_drop(meta, migrate_engine):
+    domain_table = sql.Table('domain', meta, autoload=True)
+    user_table = sql.Table('user', meta, autoload=True)
+    column = sql.Column('domain_id', sql.String(64),
+                        sql.ForeignKey('domain.id'), nullable=False)
+    column.drop(user_table)
+
+
+def downgrade_project_table_with_col_drop(meta, migrate_engine):
+    domain_table = sql.Table('domain', meta, autoload=True)
+    project_table = sql.Table('project', meta, autoload=True)
+    column = sql.Column('domain_id', sql.String(64),
+                        sql.ForeignKey('domain.id'), nullable=False)
+    column.drop(project_table)
+
+
+def upgrade(migrate_engine):
+    meta = sql.MetaData()
+    meta.bind = migrate_engine
+    session = sessionmaker(bind=migrate_engine)()
+    if migrate_engine.name in ['sqlite', 'mysql']:
+        upgrade_user_table_with_copy(meta, migrate_engine, session)
+        upgrade_project_table_with_copy(meta, migrate_engine, session)
+    else:
+        upgrade_user_table_with_col_create(meta, migrate_engine, session)
+        upgrade_project_table_with_col_create(meta, migrate_engine, session)
+    session.commit()
+    session.close()
+
+
+def downgrade(migrate_engine):
+    meta = sql.MetaData()
+    meta.bind = migrate_engine
+    session = sessionmaker(bind=migrate_engine)()
+    if migrate_engine.name in ['sqlite', 'mysql']:
+        downgrade_user_table_with_copy(meta, migrate_engine, session)
+        downgrade_project_table_with_copy(meta, migrate_engine, session)
+    else:
+        # MySQL should in theory be able to use this path, but seems to
+        # have problems dropping columns which are foreign keys
+        downgrade_user_table_with_col_drop(meta, migrate_engine)
+        downgrade_project_table_with_col_drop(meta, migrate_engine)
+    session.commit()
+    session.close()

keystone/common/sql/nova.py

@@ -18,12 +18,15 @@
 
 import uuid
 
+from keystone import config
 from keystone.common import logging
 from keystone.contrib.ec2.backends import sql as ec2_sql
 from keystone.identity.backends import sql as identity_sql
 
 
 LOG = logging.getLogger(__name__)
+CONF = config.CONF
+DEFAULT_DOMAIN_ID = CONF.identity.default_domain_id
 
 
 def import_auth(data):
@@ -51,6 +54,7 @@ def _create_projects(api, tenants):
         tenant_dict = {
             'id': _generate_uuid(),
             'name': tenant['id'],
+            'domain_id': tenant.get('domain_id', DEFAULT_DOMAIN_ID),
             'description': tenant['description'],
             'enabled': True,
         }
@@ -66,6 +70,7 @@ def _create_users(api, users):
         user_dict = {
             'id': _generate_uuid(),
             'name': user['id'],
+            'domain_id': user.get('domain_id', DEFAULT_DOMAIN_ID),
             'email': '',
             'password': user['password'],
             'enabled': True,

keystone/config.py

@@ -256,6 +256,7 @@ register_str('user_name_attribute', group='ldap', default='sn')
 register_str('user_mail_attribute', group='ldap', default='email')
 register_str('user_pass_attribute', group='ldap', default='userPassword')
 register_str('user_enabled_attribute', group='ldap', default='enabled')
+register_str('user_domain_id_attribute', group='ldap', default='domain_id')
 register_int('user_enabled_mask', group='ldap', default=0)
 register_str('user_enabled_default', group='ldap', default='True')
 register_list('user_attribute_ignore', group='ldap',
@@ -272,6 +273,7 @@ register_str('tenant_member_attribute', group='ldap', default='member')
 register_str('tenant_name_attribute', group='ldap', default='ou')
 register_str('tenant_desc_attribute', group='ldap', default='desc')
 register_str('tenant_enabled_attribute', group='ldap', default='enabled')
+register_str('tenant_domain_id_attribute', group='ldap', default='domain_id')
 register_list('tenant_attribute_ignore', group='ldap', default='')
 register_bool('tenant_allow_create', group='ldap', default=True)
 register_bool('tenant_allow_update', group='ldap', default=True)
@@ -295,6 +297,7 @@ register_str('group_id_attribute', group='ldap', default='cn')
 register_str('group_name_attribute', group='ldap', default='ou')
 register_str('group_member_attribute', group='ldap', default='member')
 register_str('group_desc_attribute', group='ldap', default='desc')
+register_str('group_domain_id_attribute', group='ldap', default='domain_id')
 register_list('group_attribute_ignore', group='ldap', default='')
 register_bool('group_allow_create', group='ldap', default=True)
 register_bool('group_allow_update', group='ldap', default=True)

keystone/identity/backends/kvs.py

@@ -67,7 +67,7 @@ class Identity(kvs.Base, identity.Driver):
         tenant_keys = filter(lambda x: x.startswith("tenant-"), self.db.keys())
         return [self.db.get(key) for key in tenant_keys]
 
-    def get_project_by_name(self, tenant_name):
+    def get_project_by_name(self, tenant_name, domain_id):
         try:
             return self.db.get('tenant_name-%s' % tenant_name)
         except exception.NotFound:
@@ -85,7 +85,7 @@ class Identity(kvs.Base, identity.Driver):
         except exception.NotFound:
             raise exception.UserNotFound(user_id=user_id)
 
-    def _get_user_by_name(self, user_name):
+    def _get_user_by_name(self, user_name, domain_id):
         try:
             return self.db.get('user_name-%s' % user_name)
         except exception.NotFound:
@@ -94,16 +94,27 @@ class Identity(kvs.Base, identity.Driver):
     def get_user(self, user_id):
         return identity.filter_user(self._get_user(user_id))
 
-    def get_user_by_name(self, user_name):
+    def get_user_by_name(self, user_name, domain_id):
-        return identity.filter_user(self._get_user_by_name(user_name))
+        return identity.filter_user(
+            self._get_user_by_name(user_name, domain_id))
 
     def get_metadata(self, user_id=None, tenant_id=None,
                      domain_id=None, group_id=None):
         try:
             if user_id:
-                return self.db.get('metadata-%s-%s' % (tenant_id, user_id))
+                if tenant_id:
+                    return self.db.get('metadata-%s-%s' % (tenant_id,
+                                                           user_id))
+                else:
+                    return self.db.get('metadata-%s-%s' % (domain_id,
+                                                           user_id))
             else:
-                return self.db.get('metadata-%s-%s' % (tenant_id, group_id))
+                if tenant_id:
+                    return self.db.get('metadata-%s-%s' % (tenant_id,
+                                                           group_id))
+                else:
+                    return self.db.get('metadata-%s-%s' % (domain_id,
+                                                           group_id))
         except exception.NotFound:
             raise exception.MetadataNotFound()
 
@@ -195,7 +206,7 @@ class Identity(kvs.Base, identity.Driver):
             raise exception.Conflict(type='user', details=msg)
 
         try:
-            self.get_user_by_name(user['name'])
+            self.get_user_by_name(user['name'], user['domain_id'])
         except exception.UserNotFound:
             pass
         else:
@@ -294,7 +305,7 @@ class Identity(kvs.Base, identity.Driver):
             raise exception.Conflict(type='tenant', details=msg)
 
         try:
-            self.get_project_by_name(tenant['name'])
+            self.get_project_by_name(tenant['name'], tenant['domain_id'])
         except exception.ProjectNotFound:
             pass
         else:
@@ -338,18 +349,22 @@ class Identity(kvs.Base, identity.Driver):
 
     def create_metadata(self, user_id, tenant_id, metadata,
                         domain_id=None, group_id=None):
-        if user_id:
+
-            self.db.set('metadata-%s-%s' % (tenant_id, user_id), metadata)
+        return self.update_metadata(user_id, tenant_id, metadata,
-        else:
+                                    domain_id, group_id)
-            self.db.set('metadata-%s-%s' % (tenant_id, group_id), metadata)
-        return metadata
 
     def update_metadata(self, user_id, tenant_id, metadata,
                         domain_id=None, group_id=None):
         if user_id:
-            self.db.set('metadata-%s-%s' % (tenant_id, user_id), metadata)
+            if tenant_id:
+                self.db.set('metadata-%s-%s' % (tenant_id, user_id), metadata)
+            else:
+                self.db.set('metadata-%s-%s' % (domain_id, user_id), metadata)
         else:
-            self.db.set('metadata-%s-%s' % (tenant_id, group_id), metadata)
+            if tenant_id:
+                self.db.set('metadata-%s-%s' % (tenant_id, group_id), metadata)
+            else:
+                self.db.set('metadata-%s-%s' % (domain_id, group_id), metadata)
         return metadata
 
     def create_role(self, role_id, role):
@@ -500,7 +515,24 @@ class Identity(kvs.Base, identity.Driver):
     # domain crud
 
     def create_domain(self, domain_id, domain):
+        try:
+            self.get_domain(domain_id)
+        except exception.DomainNotFound:
+            pass
+        else:
+            msg = 'Duplicate ID, %s.' % domain_id
+            raise exception.Conflict(type='domain', details=msg)
+
+        try:
+            self.get_domain_by_name(domain['name'])
+        except exception.DomainNotFound:
+            pass
+        else:
+            msg = 'Duplicate name, %s.' % domain['name']
+            raise exception.Conflict(type='domain', details=msg)
+
         self.db.set('domain-%s' % domain_id, domain)
+        self.db.set('domain_name-%s' % domain['name'], domain)
         domain_list = set(self.db.get('domain_list', []))
         domain_list.add(domain_id)
         self.db.set('domain_list', list(domain_list))
@@ -510,14 +542,30 @@ class Identity(kvs.Base, identity.Driver):
         return self.db.get('domain_list', [])
 
     def get_domain(self, domain_id):
-        return self.db.get('domain-%s' % domain_id)
+        try:
+            return self.db.get('domain-%s' % domain_id)
+        except exception.NotFound:
+            raise exception.DomainNotFound(domain_id=domain_id)
+
+    def get_domain_by_name(self, domain_name):
+        try:
+            return self.db.get('domain_name-%s' % domain_name)
+        except exception.NotFound:
+            raise exception.DomainNotFound(domain_id=domain_name)
 
     def update_domain(self, domain_id, domain):
+        orig_domain = self.get_domain(domain_id)
+        domain['id'] = domain_id
         self.db.set('domain-%s' % domain_id, domain)
+        self.db.set('domain_name-%s' % domain['name'], domain)
+        if domain['name'] != orig_domain['name']:
+            self.db.delete('domain_name-%s' % orig_domain['name'])
         return domain
 
     def delete_domain(self, domain_id):
+        domain = self.get_domain(domain_id)
         self.db.delete('domain-%s' % domain_id)
+        self.db.delete('domain_name-%s' % domain['name'])
         domain_list = set(self.db.get('domain_list', []))
         domain_list.remove(domain_id)
         self.db.set('domain_list', list(domain_list))

keystone/identity/backends/ldap/core.py

@@ -106,7 +106,9 @@ class Identity(identity.Driver):
     def get_projects(self):
         return self.project.get_all()
 
-    def get_project_by_name(self, tenant_name):
+    def get_project_by_name(self, tenant_name, domain_id):
+        # TODO(henry-nash): Use domain_id once domains are implemented
+        # in LDAP backend
         try:
             return self.project.get_by_name(tenant_name)
         except exception.NotFound:
@@ -124,7 +126,9 @@ class Identity(identity.Driver):
     def list_users(self):
         return self.user.get_all()
 
-    def get_user_by_name(self, user_name):
+    def get_user_by_name(self, user_name, domain_id):
+        # TODO(henry-nash): Use domain_id once domains are implemented
+        # in LDAP backend
         try:
             return identity.filter_user(self.user.get_by_name(user_name))
         except exception.NotFound:
@@ -353,7 +357,8 @@ class UserApi(common_ldap.BaseLdap, ApiShimMixin):
     attribute_mapping = {'password': 'userPassword',
                          'email': 'mail',
                          'name': 'sn',
-                         'enabled': 'enabled'}
+                         'enabled': 'enabled',
+                         'domain_id': 'domain_id'}
 
     model = models.User
 
@@ -363,6 +368,8 @@ class UserApi(common_ldap.BaseLdap, ApiShimMixin):
         self.attribute_mapping['email'] = conf.ldap.user_mail_attribute
         self.attribute_mapping['password'] = conf.ldap.user_pass_attribute
         self.attribute_mapping['enabled'] = conf.ldap.user_enabled_attribute
+        self.attribute_mapping['domain_id'] = (
+            conf.ldap.user_domain_id_attribute)
         self.enabled_mask = conf.ldap.user_enabled_mask
         self.enabled_default = conf.ldap.user_enabled_default
         self.attribute_ignore = (getattr(conf.ldap, 'user_attribute_ignore')
@@ -510,7 +517,8 @@ class ProjectApi(common_ldap.BaseLdap, ApiShimMixin):
     attribute_mapping = {'name': 'ou',
                          'description': 'desc',
                          'tenantId': 'cn',
-                         'enabled': 'enabled'}
+                         'enabled': 'enabled',
+                         'domain_id': 'domain_id'}
     model = models.Project
 
     def __init__(self, conf):
@@ -519,6 +527,8 @@ class ProjectApi(common_ldap.BaseLdap, ApiShimMixin):
         self.attribute_mapping['name'] = conf.ldap.tenant_name_attribute
         self.attribute_mapping['description'] = conf.ldap.tenant_desc_attribute
         self.attribute_mapping['enabled'] = conf.ldap.tenant_enabled_attribute
+        self.attribute_mapping['domain_id'] = (
+            conf.ldap.tenant_domain_id_attribute)
         self.member_attribute = (getattr(conf.ldap, 'tenant_member_attribute')
                                  or self.DEFAULT_MEMBER_ATTRIBUTE)
         self.attribute_ignore = (getattr(conf.ldap, 'tenant_attribute_ignore')
@@ -1070,7 +1080,8 @@ class GroupApi(common_ldap.BaseLdap, ApiShimMixin):
     options_name = 'group'
     attribute_mapping = {'name': 'ou',
                          'description': 'desc',
-                         'groupId': 'cn'}
+                         'groupId': 'cn',
+                         'domain_id': 'domain_id'}
     model = models.Group
 
     def __init__(self, conf):
@@ -1078,6 +1089,8 @@ class GroupApi(common_ldap.BaseLdap, ApiShimMixin):
         self.api = ApiShim(conf)
         self.attribute_mapping['name'] = conf.ldap.group_name_attribute
         self.attribute_mapping['description'] = conf.ldap.group_desc_attribute
+        self.attribute_mapping['domain_id'] = (
+            conf.ldap.group_domain_id_attribute)
         self.member_attribute = (getattr(conf.ldap, 'group_member_attribute')
                                  or self.DEFAULT_MEMBER_ATTRIBUTE)
         self.attribute_ignore = (getattr(conf.ldap, 'group_attribute_ignore')

keystone/identity/backends/pam.py

@@ -74,13 +74,17 @@ class PamIdentity(identity.Driver):
     def get_project(self, tenant_id):
         return {'id': tenant_id, 'name': tenant_id}
 
-    def get_project_by_name(self, tenant_name):
+    def get_project_by_name(self, tenant_name, domain_id):
+        # TODO(henry-nash): Used domain_id once domains are implemented
+        # in LDAP backend
         return {'id': tenant_name, 'name': tenant_name}
 
     def get_user(self, user_id):
         return {'id': user_id, 'name': user_id}
 
-    def get_user_by_name(self, user_name):
+    def get_user_by_name(self, user_name, domain_id):
+        # TODO(henry-nash): Used domain_id once domains are implemented
+        # in LDAP backend
         return {'id': user_name, 'name': user_name}
 
     def get_role(self, role_id):

keystone/identity/backends/sql.py

@@ -39,9 +39,11 @@ def handle_conflicts(type='object'):
 
 class User(sql.ModelBase, sql.DictBase):
     __tablename__ = 'user'
-    attributes = ['id', 'name', 'password', 'enabled']
+    attributes = ['id', 'name', 'domain_id', 'password', 'enabled']
     id = sql.Column(sql.String(64), primary_key=True)
     name = sql.Column(sql.String(64), unique=True, nullable=False)
+    domain_id = sql.Column(sql.String(64), sql.ForeignKey('domain.id'),
+                           nullable=False)
     password = sql.Column(sql.String(128))
     enabled = sql.Column(sql.Boolean)
     extra = sql.Column(sql.JsonBlob())
@@ -52,7 +54,8 @@ class Group(sql.ModelBase, sql.DictBase):
     attributes = ['id', 'name', 'domain_id']
     id = sql.Column(sql.String(64), primary_key=True)
     name = sql.Column(sql.String(64), unique=True, nullable=False)
-    domain_id = sql.Column(sql.String(64), sql.ForeignKey('domain.id'))
+    domain_id = sql.Column(sql.String(64), sql.ForeignKey('domain.id'),
+                           nullable=False)
     description = sql.Column(sql.Text())
     extra = sql.Column(sql.JsonBlob())
 
@@ -82,9 +85,11 @@ class Domain(sql.ModelBase, sql.DictBase):
 # TODO(dolph): rename to Project
 class Project(sql.ModelBase, sql.DictBase):
     __tablename__ = 'project'
-    attributes = ['id', 'name']
+    attributes = ['id', 'name', 'domain_id']
     id = sql.Column(sql.String(64), primary_key=True)
     name = sql.Column(sql.String(64), unique=True, nullable=False)
+    domain_id = sql.Column(sql.String(64), sql.ForeignKey('domain.id'),
+                           nullable=False)
     description = sql.Column(sql.Text())
     enabled = sql.Column(sql.Boolean)
     extra = sql.Column(sql.JsonBlob())
@@ -222,12 +227,16 @@ class Identity(sql.Base, identity.Driver):
             raise exception.ProjectNotFound(project_id=tenant_id)
         return tenant_ref.to_dict()
 
-    def get_project_by_name(self, tenant_name):
+    def get_project_by_name(self, tenant_name, domain_id):
         session = self.get_session()
-        tenant_ref = session.query(Project).filter_by(name=tenant_name).first()
+        query = session.query(Project)
-        if not tenant_ref:
+        query = query.filter_by(name=tenant_name)
+        query = query.filter_by(domain_id=domain_id)
+        try:
+            project_ref = query.one()
+        except sql.NotFound:
             raise exception.ProjectNotFound(project_id=tenant_name)
-        return tenant_ref.to_dict()
+        return project_ref.to_dict()
 
     def get_project_users(self, tenant_id):
         session = self.get_session()
@@ -484,7 +493,8 @@ class Identity(sql.Base, identity.Driver):
             tenant_ref = session.query(Project).filter_by(id=tenant_id).one()
         except sql.NotFound:
             raise exception.ProjectNotFound(project_id=tenant_id)
-
+        # FIXME(henry-nash) Think about how we detect potential name clash
+        # when we move domains
         with session.begin():
             old_project_dict = tenant_ref.to_dict()
             for k in tenant:
@@ -603,6 +613,14 @@ class Identity(sql.Base, identity.Driver):
             raise exception.DomainNotFound(domain_id=domain_id)
         return ref.to_dict()
 
+    def get_domain_by_name(self, domain_name):
+        session = self.get_session()
+        try:
+            ref = session.query(Domain).filter_by(name=domain_name).one()
+        except sql.NotFound:
+            raise exception.DomainNotFound(domain_id=domain_name)
+        return ref.to_dict()
+
     @handle_conflicts(type='domain')
     def update_domain(self, domain_id, domain):
         session = self.get_session()
@@ -674,18 +692,23 @@ class Identity(sql.Base, identity.Driver):
             raise exception.UserNotFound(user_id=user_id)
         return user_ref.to_dict()
 
-    def _get_user_by_name(self, user_name):
+    def _get_user_by_name(self, user_name, domain_id):
         session = self.get_session()
-        user_ref = session.query(User).filter_by(name=user_name).first()
+        query = session.query(User)
-        if not user_ref:
+        query = query.filter_by(name=user_name)
+        query = query.filter_by(domain_id=domain_id)
+        try:
+            user_ref = query.one()
+        except sql.NotFound:
             raise exception.UserNotFound(user_id=user_name)
         return user_ref.to_dict()
 
     def get_user(self, user_id):
         return identity.filter_user(self._get_user(user_id))
 
-    def get_user_by_name(self, user_name):
+    def get_user_by_name(self, user_name, domain_id):
-        return identity.filter_user(self._get_user_by_name(user_name))
+        return identity.filter_user(
+            self._get_user_by_name(user_name, domain_id))
 
     @handle_conflicts(type='user')
     def update_user(self, user_id, user):
@@ -694,6 +717,8 @@ class Identity(sql.Base, identity.Driver):
         session = self.get_session()
         if 'id' in user and user_id != user['id']:
             raise exception.ValidationError('Cannot change user ID')
+        # FIXME(henry-nash) Think about how we detect potential name clash
+        # when we move domains
         with session.begin():
             user_ref = session.query(User).filter_by(id=user_id).first()
             if user_ref is None:
@@ -826,6 +851,8 @@ class Identity(sql.Base, identity.Driver):
     @handle_conflicts(type='group')
     def update_group(self, group_id, group):
         session = self.get_session()
+        # FIXME(henry-nash) Think about how we detect potential name clash
+        # when we move domains
         with session.begin():
             ref = session.query(Group).filter_by(id=group_id).first()
             if ref is None:

keystone/identity/controllers.py

@@ -27,7 +27,7 @@ from keystone import exception
 
 
 CONF = config.CONF
-DEFAULT_DOMAIN_ID = CONF['identity']['default_domain_id']
+DEFAULT_DOMAIN_ID = CONF.identity.default_domain_id
 LOG = logging.getLogger(__name__)
 
 
@@ -40,6 +40,8 @@ class Tenant(controller.V2Controller):
 
         self.assert_admin(context)
         tenant_refs = self.identity_api.get_projects(context)
+        for tenant_ref in tenant_refs:
+            tenant_ref = self._filter_domain_id(tenant_ref)
         params = {
             'limit': context['query_string'].get('limit'),
             'marker': context['query_string'].get('marker'),
@@ -67,9 +69,9 @@ class Tenant(controller.V2Controller):
             context, user_ref['id'])
         tenant_refs = []
         for tenant_id in tenant_ids:
-            tenant_refs.append(self.identity_api.get_project(
+            ref = self.identity_api.get_project(
-                context=context,
+                context=context, tenant_id=tenant_id)
-                tenant_id=tenant_id))
+            tenant_refs.append(self._filter_domain_id(ref))
         params = {
             'limit': context['query_string'].get('limit'),
             'marker': context['query_string'].get('marker'),
@@ -79,12 +81,14 @@ class Tenant(controller.V2Controller):
     def get_project(self, context, tenant_id):
         # TODO(termie): this stuff should probably be moved to middleware
         self.assert_admin(context)
-        return {'tenant': self.identity_api.get_project(context, tenant_id)}
+        ref = self.identity_api.get_project(context, tenant_id)
+        return {'tenant': self._filter_domain_id(ref)}
 
     def get_project_by_name(self, context, tenant_name):
         self.assert_admin(context)
-        return {'tenant': self.identity_api.get_project_by_name(
+        ref = self.identity_api.get_project_by_name(
-            context, tenant_name)}
+            context, tenant_name, DEFAULT_DOMAIN_ID)
+        return {'tenant': self._filter_domain_id(ref)}
 
     # CRUD Extension
     def create_project(self, context, tenant):
@@ -97,13 +101,18 @@ class Tenant(controller.V2Controller):
         self.assert_admin(context)
         tenant_ref['id'] = tenant_ref.get('id', uuid.uuid4().hex)
         tenant = self.identity_api.create_project(
-            context, tenant_ref['id'], tenant_ref)
+            context, tenant_ref['id'],
-        return {'tenant': tenant}
+            self._normalize_domain_id(context, tenant_ref))
+        return {'tenant': self._filter_domain_id(tenant)}
 
     def update_project(self, context, tenant_id, tenant):
         self.assert_admin(context)
+        # Remove domain_id if specified - a v2 api caller should not
+        # be specifying that
+        clean_tenant = tenant.copy()
+        clean_tenant.pop('domain_id', None)
         tenant_ref = self.identity_api.update_project(
-            context, tenant_id, tenant)
+            context, tenant_id, clean_tenant)
         return {'tenant': tenant_ref}
 
     def delete_project(self, context, tenant_id):
@@ -113,6 +122,8 @@ class Tenant(controller.V2Controller):
     def get_project_users(self, context, tenant_id, **kw):
         self.assert_admin(context)
         user_refs = self.identity_api.get_project_users(context, tenant_id)
+        for user_ref in user_refs:
+            self._filter_domain_id(user_ref)
         return {'users': user_refs}
 
     def _format_project_list(self, tenant_refs, **kwargs):
@@ -153,7 +164,8 @@ class Tenant(controller.V2Controller):
 class User(controller.V2Controller):
     def get_user(self, context, user_id):
         self.assert_admin(context)
-        return {'user': self.identity_api.get_user(context, user_id)}
+        ref = self.identity_api.get_user(context, user_id)
+        return {'user': self._filter_domain_id(ref)}
 
     def get_users(self, context):
         # NOTE(termie): i can't imagine that this really wants all the data
@@ -163,11 +175,16 @@ class User(controller.V2Controller):
                 context, context['query_string'].get('name'))
 
         self.assert_admin(context)
-        return {'users': self.identity_api.list_users(context)}
+        user_list = self.identity_api.list_users(context)
+        for x in user_list:
+            self._filter_domain_id(x)
+        return {'users': user_list}
 
     def get_user_by_name(self, context, user_name):
         self.assert_admin(context)
-        return {'user': self.identity_api.get_user_by_name(context, user_name)}
+        ref = self.identity_api.get_user_by_name(
+            context, user_name, DEFAULT_DOMAIN_ID)
+        return {'user': self._filter_domain_id(ref)}
 
     # CRUD extension
     def create_user(self, context, user):
@@ -178,18 +195,20 @@ class User(controller.V2Controller):
             msg = 'Name field is required and cannot be empty'
             raise exception.ValidationError(message=msg)
 
-        tenant_id = user.get('tenantId', None)
+        default_tenant_id = user.get('tenantId', None)
-        if (tenant_id is not None
+        if (default_tenant_id is not None
-                and self.identity_api.get_project(context, tenant_id) is None):
+                and self.identity_api.get_project(context,
-            raise exception.ProjectNotFound(project_id=tenant_id)
+                                                  default_tenant_id) is None):
+            raise exception.ProjectNotFound(project_id=default_tenant_id)
         user_id = uuid.uuid4().hex
-        user_ref = user.copy()
+        user_ref = self._normalize_domain_id(context, user.copy())
         user_ref['id'] = user_id
         new_user_ref = self.identity_api.create_user(
             context, user_id, user_ref)
-        if tenant_id:
+        if default_tenant_id:
-            self.identity_api.add_user_to_project(context, tenant_id, user_id)
+            self.identity_api.add_user_to_project(context,
-        return {'user': new_user_ref}
+                                                  default_tenant_id, user_id)
+        return {'user': self._filter_domain_id(new_user_ref)}
 
     def update_user(self, context, user_id, user):
         # NOTE(termie): this is really more of a patch than a put
@@ -206,7 +225,7 @@ class User(controller.V2Controller):
                 # backends that can't list tokens for users
                 LOG.warning('User %s status has changed, but existing tokens '
                             'remain valid' % user_id)
-        return {'user': user_ref}
+        return {'user': self._filter_domain_id(user_ref)}
 
     def delete_user(self, context, user_id):
         self.assert_admin(context)
@@ -222,8 +241,9 @@ class User(controller.V2Controller):
         """Update the default tenant."""
         self.assert_admin(context)
         # ensure that we're a member of that tenant
-        tenant_id = user.get('tenantId')
+        default_tenant_id = user.get('tenantId')
-        self.identity_api.add_user_to_project(context, tenant_id, user_id)
+        self.identity_api.add_user_to_project(context,
+                                              default_tenant_id, user_id)
         return self.update_user(context, user_id, user)
 
 
@@ -403,6 +423,7 @@ class DomainV3(controller.V3Controller):
     @controller.protected
     def list_domains(self, context):
         refs = self.identity_api.list_domains(context)
+        refs = self._filter_by_attribute(context, refs, 'name')
         return DomainV3.wrap_collection(context, refs)
 
     @controller.protected
@@ -456,6 +477,17 @@ class DomainV3(controller.V3Controller):
 
         return self.identity_api.delete_domain(context, domain_id)
 
+    def _get_domain_by_name(self, context, domain_name):
+        """Get the domain via its unique name.
+
+        For use by token authentication - not for hooking to the identity
+        router as a public api.
+
+        """
+        ref = self.identity_api.get_domain_by_name(
+            context, domain_name)
+        return {'domain': ref}
+
 
 class ProjectV3(controller.V3Controller):
     collection_name = 'projects'
@@ -464,6 +496,7 @@ class ProjectV3(controller.V3Controller):
     @controller.protected
     def create_project(self, context, project):
         ref = self._assign_unique_id(self._normalize_dict(project))
+        ref = self._normalize_domain_id(context, ref)
         ref = self.identity_api.create_project(context, ref['id'], ref)
         return ProjectV3.wrap_member(context, ref)
 
@@ -501,6 +534,7 @@ class UserV3(controller.V3Controller):
     @controller.protected
     def create_user(self, context, user):
         ref = self._assign_unique_id(self._normalize_dict(user))
+        ref = self._normalize_domain_id(context, ref)
         ref = self.identity_api.create_user(context, ref['id'], ref)
         return UserV3.wrap_member(context, ref)
 
@@ -560,6 +594,7 @@ class GroupV3(controller.V3Controller):
     @controller.protected
     def create_group(self, context, group):
         ref = self._assign_unique_id(self._normalize_dict(group))
+        ref = self._normalize_domain_id(context, ref)
         ref = self.identity_api.create_group(context, ref['id'], ref)
         return GroupV3.wrap_member(context, ref)
 

keystone/identity/core.py

@@ -29,7 +29,9 @@ LOG = logging.getLogger(__name__)
 
 
 def filter_user(user_ref):
-    """Filter out private items in a user dict ('password' and 'tenants')
+    """Filter out private items in a user dict.
+
+    'password', 'tenants' and 'groups' are never returned.
 
     :returns: user_ref
 
@@ -81,7 +83,7 @@ class Driver(object):
         """
         raise exception.NotImplemented()
 
-    def get_project_by_name(self, tenant_name):
+    def get_project_by_name(self, tenant_name, domain_id):
         """Get a tenant by name.
 
         :returns: tenant_ref
@@ -90,7 +92,7 @@ class Driver(object):
         """
         raise exception.NotImplemented()
 
-    def get_user_by_name(self, user_name):
+    def get_user_by_name(self, user_name, domain_id):
         """Get a user by name.
 
         :returns: user_ref
@@ -252,7 +254,16 @@ class Driver(object):
     def get_domain(self, domain_id):
         """Get a domain by ID.
 
-        :returns: user_ref
+        :returns: domain_ref
+        :raises: keystone.exception.DomainNotFound
+
+        """
+        raise exception.NotImplemented()
+
+    def get_domain_by_name(self, domain_name):
+        """Get a domain by name.
+
+        :returns: domain_ref
         :raises: keystone.exception.DomainNotFound
 
         """

keystone/token/controllers.py

@@ -13,6 +13,7 @@ from keystone.token import core
 
 CONF = config.CONF
 LOG = logging.getLogger(__name__)
+DEFAULT_DOMAIN_ID = CONF.identity.default_domain_id
 
 
 class ExternalAuthNotApplicable(Exception):
@@ -64,19 +65,26 @@ class Auth(controller.V2Controller):
 
         if "token" in auth:
             # Try to authenticate using a token
-            auth_token_data, auth_info = self._authenticate_token(
+            auth_info = self._authenticate_token(
                 context, auth)
         else:
             # Try external authentication
             try:
-                auth_token_data, auth_info = self._authenticate_external(
+                auth_info = self._authenticate_external(
                     context, auth)
             except ExternalAuthNotApplicable:
                 # Try local authentication
-                auth_token_data, auth_info = self._authenticate_local(
+                auth_info = self._authenticate_local(
                     context, auth)
 
-        user_ref, tenant_ref, metadata_ref = auth_info
+        user_ref, tenant_ref, metadata_ref, expiry = auth_info
+        user_ref = self._filter_domain_id(user_ref)
+        if tenant_ref:
+            tenant_ref = self._filter_domain_id(tenant_ref)
+        auth_token_data = self._get_auth_token_data(user_ref,
+                                                    tenant_ref,
+                                                    metadata_ref,
+                                                    expiry)
 
         # If the user is disabled don't allow them to authenticate
         if not user_ref.get('enabled', True):
@@ -202,21 +210,15 @@ class Auth(controller.V2Controller):
         tenant_ref = self._get_project_ref(context, user_id, tenant_id)
         metadata_ref = self._get_metadata_ref(context, user_id, tenant_id)
 
+        # TODO (henry-nash) If no tenant was specified, instead check
+        # for a domain and find any related user/group roles
+
         self._append_roles(metadata_ref,
                            self._get_group_metadata_ref(
                                context, user_id, tenant_id))
 
-        self._append_roles(metadata_ref,
-                           self._get_domain_metadata_ref(
-                               context, user_id, tenant_id))
-
         expiry = old_token_ref['expires']
-        auth_token_data = self._get_auth_token_data(current_user_ref,
+        return (current_user_ref, tenant_ref, metadata_ref, expiry)
-                                                    tenant_ref,
-                                                    metadata_ref,
-                                                    expiry)
-
-        return auth_token_data, (current_user_ref, tenant_ref, metadata_ref)
 
     def _authenticate_local(self, context, auth):
         """Try to authenticate against the identity backend.
@@ -256,7 +258,8 @@ class Auth(controller.V2Controller):
         if username:
             try:
                 user_ref = self.identity_api.get_user_by_name(
-                    context=context, user_name=username)
+                    context=context, user_name=username,
+                    domain_id=DEFAULT_DOMAIN_ID)
                 user_id = user_ref['id']
             except exception.UserNotFound as e:
                 raise exception.Unauthorized(e)
@@ -273,21 +276,19 @@ class Auth(controller.V2Controller):
             raise exception.Unauthorized(e)
         (user_ref, tenant_ref, metadata_ref) = auth_info
 
+        # By now we will have authorized and if a tenant/project was
+        # specified, we will have obtained its metadata.  In this case
+        # we just need to add in any group roles.
+        #
+        # TODO (henry-nash) If no tenant was specified, instead check
+        # for a domain and find any related user/group roles
+
         self._append_roles(metadata_ref,
                            self._get_group_metadata_ref(
                                context, user_id, tenant_id))
 
-        self._append_roles(metadata_ref,
-                           self._get_domain_metadata_ref(
-                               context, user_id, tenant_id))
-
         expiry = core.default_expire_time()
-        auth_token_data = self._get_auth_token_data(user_ref,
+        return (user_ref, tenant_ref, metadata_ref, expiry)
-                                                    tenant_ref,
-                                                    metadata_ref,
-                                                    expiry)
-
-        return auth_token_data, (user_ref, tenant_ref, metadata_ref)
 
     def _authenticate_external(self, context, auth):
         """Try to authenticate an external user via REMOTE_USER variable.
@@ -300,7 +301,8 @@ class Auth(controller.V2Controller):
         username = context['REMOTE_USER']
         try:
             user_ref = self.identity_api.get_user_by_name(
-                context=context, user_name=username)
+                context=context, user_name=username,
+                domain_id=DEFAULT_DOMAIN_ID)
             user_id = user_ref['id']
         except exception.UserNotFound as e:
             raise exception.Unauthorized(e)
@@ -310,21 +312,15 @@ class Auth(controller.V2Controller):
         tenant_ref = self._get_project_ref(context, user_id, tenant_id)
         metadata_ref = self._get_metadata_ref(context, user_id, tenant_id)
 
+        # TODO (henry-nash) If no tenant was specified, instead check
+        # for a domain and find any related user/group roles
+
         self._append_roles(metadata_ref,
                            self._get_group_metadata_ref(
                                context, user_id, tenant_id))
 
-        self._append_roles(metadata_ref,
-                           self._get_domain_metadata_ref(
-                               context, user_id, tenant_id))
-
         expiry = core.default_expire_time()
-        auth_token_data = self._get_auth_token_data(user_ref,
+        return (user_ref, tenant_ref, metadata_ref, expiry)
-                                                    tenant_ref,
-                                                    metadata_ref,
-                                                    expiry)
-
-        return auth_token_data, (user_ref, tenant_ref, metadata_ref)
 
     def _get_auth_token_data(self, user, tenant, metadata, expiry):
         return dict(dict(user=user,
@@ -350,12 +346,32 @@ class Auth(controller.V2Controller):
         if tenant_name:
             try:
                 tenant_ref = self.identity_api.get_project_by_name(
-                    context=context, tenant_name=tenant_name)
+                    context=context, tenant_name=tenant_name,
+                    domain_id=DEFAULT_DOMAIN_ID)
                 tenant_id = tenant_ref['id']
             except exception.ProjectNotFound as e:
                 raise exception.Unauthorized(e)
         return tenant_id
 
+    def _get_domain_id_from_auth(self, context, auth):
+        """Extract domain information from v3 auth dict.
+
+        Returns a valid domain_id if it exists, or None if not specified.
+        """
+        # FIXME(henry-nash): This is a placeholder that needs to be
+        # only called in the v3 context, and the auth.get calls
+        # converted to the v3 format
+        domain_id = auth.get('domainId', None)
+        domain_name = auth.get('domainName', None)
+        if domain_name:
+            try:
+                domain_ref = self.identity_api._get_domain_by_name(
+                    context=context, domain_name=domain_name)
+                domain_id = domain_ref['id']
+            except exception.DomainNotFound as e:
+                raise exception.Unauthorized(e)
+        return domain_id
+
     def _get_project_ref(self, context, user_id, tenant_id):
         """Returns the tenant_ref for the user's tenant"""
         tenant_ref = None
@@ -375,43 +391,32 @@ class Auth(controller.V2Controller):
         return tenant_ref
 
     def _get_metadata_ref(self, context, user_id=None, tenant_id=None,
-                          group_id=None):
+                          domain_id=None, group_id=None):
-        """Returns the metadata_ref for a user or group in a tenant"""
+        """Returns metadata_ref for a user or group in a tenant or domain"""
-        metadata_ref = {}
-        if tenant_id:
-            try:
-                if user_id:
-                    metadata_ref = self.identity_api.get_metadata(
-                        context=context,
-                        user_id=user_id,
-                        tenant_id=tenant_id)
-                elif group_id:
-                    metadata_ref = self.identity_api.get_metadata(
-                        context=context,
-                        group_id=group_id,
-                        tenant_id=tenant_id)
-            except exception.MetadataNotFound:
-                metadata_ref = {}
 
+        metadata_ref = {}
+        if (user_id or group_id) and (tenant_id or domain_id):
+            try:
+                metadata_ref = self.identity_api.get_metadata(
+                    context=context, user_id=user_id, tenant_id=tenant_id,
+                    domain_id=domain_id, group_id=group_id)
+            except exception.MetadataNotFound:
+                pass
         return metadata_ref
 
-    def _get_group_metadata_ref(self, context, user_id, tenant_id):
+    def _get_group_metadata_ref(self, context, user_id,
-        """Return any metadata for this project due to group grants"""
+                                tenant_id=None, domain_id=None):
+        """Return any metadata for this project/domain due to group grants"""
         group_refs = self.identity_api.list_groups_for_user(context=context,
                                                             user_id=user_id)
         metadata_ref = {}
         for x in group_refs:
             metadata_ref.update(self._get_metadata_ref(context,
                                                        group_id=x['id'],
-                                                       tenant_id=tenant_id))
+                                                       tenant_id=tenant_id,
+                                                       domain_id=domain_id))
         return metadata_ref
 
-    def _get_domain_metadata_ref(self, context, user_id, tenant_id):
-        """Return any metadata for this project due to domain grants"""
-        # TODO (henry-nashe) Get the domain for this tenant...and then see if
-        # any domain grants apply.  Bug #1093248
-        return {}
-
     def _append_roles(self, metadata, additional_metadata):
         """
         Update the roles in metadata to be the union of the roles from

tests/default_fixtures.py

@@ -17,13 +17,21 @@
 # NOTE(dolph): please try to avoid additional fixtures if possible; test suite
 #              performance may be negatively affected.
 
+from keystone import config
+
+
+DEFAULT_DOMAIN_ID = config.CONF.identity.default_domain_id
+
+
 TENANTS = [
     {
         'id': 'bar',
         'name': 'BAR',
+        'domain_id': DEFAULT_DOMAIN_ID,
     }, {
         'id': 'baz',
         'name': 'BAZ',
+        'domain_id': DEFAULT_DOMAIN_ID,
         'description': 'description',
         'enabled': True,
     }
@@ -34,11 +42,13 @@ USERS = [
     {
         'id': 'foo',
         'name': 'FOO',
+        'domain_id': DEFAULT_DOMAIN_ID,
         'password': 'foo2',
         'tenants': ['bar']
     }, {
         'id': 'two',
         'name': 'TWO',
+        'domain_id': DEFAULT_DOMAIN_ID,
         'password': 'two2',
         'email': 'two@example.com',
         'enabled': True,
@@ -47,6 +57,7 @@ USERS = [
     }, {
         'id': 'badguy',
         'name': 'BadGuy',
+        'domain_id': DEFAULT_DOMAIN_ID,
         'password': 'bad',
         'email': 'bad@guy.com',
         'enabled': False,

tests/test_backend.py

@@ -21,9 +21,14 @@ import uuid
 from keystone.catalog import core
 from keystone import exception
 from keystone.openstack.common import timeutils
+from keystone import config
 from keystone import test
 
 
+CONF = config.CONF
+DEFAULT_DOMAIN_ID = CONF.identity.default_domain_id
+
+
 class IdentityTests(object):
     def test_authenticate_bad_user(self):
         self.assertRaises(AssertionError,
@@ -85,6 +90,7 @@ class IdentityTests(object):
         user = {
             'id': 'no_meta',
             'name': 'NO_META',
+            'domain_id': DEFAULT_DOMAIN_ID,
             'password': 'no_meta2',
         }
         self.identity_api.create_user(user['id'], user)
@@ -118,13 +124,15 @@ class IdentityTests(object):
 
     def test_get_project_by_name(self):
         tenant_ref = self.identity_api.get_project_by_name(
-            tenant_name=self.tenant_bar['name'])
+            tenant_name=self.tenant_bar['name'],
+            domain_id=DEFAULT_DOMAIN_ID)
         self.assertDictEqual(tenant_ref, self.tenant_bar)
 
     def test_get_project_by_name_404(self):
         self.assertRaises(exception.ProjectNotFound,
-                          self.identity_api.get_project,
+                          self.identity_api.get_project_by_name,
-                          tenant_id=uuid.uuid4().hex)
+                          tenant_name=uuid.uuid4().hex,
+                          domain_id=DEFAULT_DOMAIN_ID)
 
     def test_get_project_users_404(self):
         self.assertRaises(exception.ProjectNotFound,
@@ -146,7 +154,8 @@ class IdentityTests(object):
 
     def test_get_user_by_name(self):
         user_ref = self.identity_api.get_user_by_name(
-            user_name=self.user_foo['name'])
+            user_name=self.user_foo['name'],
+            domain_id=DEFAULT_DOMAIN_ID)
         # NOTE(termie): the password field is left in user_foo to make
         #               it easier to authenticate in tests, but should
         #               not be returned by the api
@@ -156,7 +165,8 @@ class IdentityTests(object):
     def test_get_user_by_name_404(self):
         self.assertRaises(exception.UserNotFound,
                           self.identity_api.get_user_by_name,
-                          user_name=uuid.uuid4().hex)
+                          user_name=uuid.uuid4().hex,
+                          domain_id=DEFAULT_DOMAIN_ID)
 
     def test_get_metadata(self):
         metadata_ref = self.identity_api.get_metadata(
@@ -217,6 +227,7 @@ class IdentityTests(object):
     def test_create_duplicate_user_id_fails(self):
         user = {'id': 'fake1',
                 'name': 'fake1',
+                'domain_id': DEFAULT_DOMAIN_ID,
                 'password': 'fakepass',
                 'tenants': ['bar']}
         self.identity_api.create_user('fake1', user)
@@ -229,6 +240,7 @@ class IdentityTests(object):
     def test_create_duplicate_user_name_fails(self):
         user = {'id': 'fake1',
                 'name': 'fake1',
+                'domain_id': DEFAULT_DOMAIN_ID,
                 'password': 'fakepass',
                 'tenants': ['bar']}
         self.identity_api.create_user('fake1', user)
@@ -241,10 +253,12 @@ class IdentityTests(object):
     def test_rename_duplicate_user_name_fails(self):
         user1 = {'id': 'fake1',
                  'name': 'fake1',
+                 'domain_id': DEFAULT_DOMAIN_ID,
                  'password': 'fakepass',
                  'tenants': ['bar']}
         user2 = {'id': 'fake2',
                  'name': 'fake2',
+                 'domain_id': DEFAULT_DOMAIN_ID,
                  'password': 'fakepass',
                  'tenants': ['bar']}
         self.identity_api.create_user('fake1', user1)
@@ -258,6 +272,7 @@ class IdentityTests(object):
     def test_update_user_id_fails(self):
         user = {'id': 'fake1',
                 'name': 'fake1',
+                'domain_id': DEFAULT_DOMAIN_ID,
                 'password': 'fakepass',
                 'tenants': ['bar']}
         self.identity_api.create_user('fake1', user)
@@ -273,7 +288,8 @@ class IdentityTests(object):
                           'fake2')
 
     def test_create_duplicate_project_id_fails(self):
-        tenant = {'id': 'fake1', 'name': 'fake1'}
+        tenant = {'id': 'fake1', 'name': 'fake1',
+                  'domain_id': DEFAULT_DOMAIN_ID}
         self.identity_api.create_project('fake1', tenant)
         tenant['name'] = 'fake2'
         self.assertRaises(exception.Conflict,
@@ -282,7 +298,8 @@ class IdentityTests(object):
                           tenant)
 
     def test_create_duplicate_project_name_fails(self):
-        tenant = {'id': 'fake1', 'name': 'fake'}
+        tenant = {'id': 'fake1', 'name': 'fake',
+                  'domain_id': DEFAULT_DOMAIN_ID}
         self.identity_api.create_project('fake1', tenant)
         tenant['id'] = 'fake2'
         self.assertRaises(exception.Conflict,
@@ -291,8 +308,10 @@ class IdentityTests(object):
                           tenant)
 
     def test_rename_duplicate_project_name_fails(self):
-        tenant1 = {'id': 'fake1', 'name': 'fake1'}
+        tenant1 = {'id': 'fake1', 'name': 'fake1',
-        tenant2 = {'id': 'fake2', 'name': 'fake2'}
+                   'domain_id': DEFAULT_DOMAIN_ID}
+        tenant2 = {'id': 'fake2', 'name': 'fake2',
+                   'domain_id': DEFAULT_DOMAIN_ID}
         self.identity_api.create_project('fake1', tenant1)
         self.identity_api.create_project('fake2', tenant2)
         tenant2['name'] = 'fake1'
@@ -302,7 +321,8 @@ class IdentityTests(object):
                           tenant2)
 
     def test_update_project_id_does_nothing(self):
-        tenant = {'id': 'fake1', 'name': 'fake1'}
+        tenant = {'id': 'fake1', 'name': 'fake1',
+                  'domain_id': DEFAULT_DOMAIN_ID}
         self.identity_api.create_project('fake1', tenant)
         tenant['id'] = 'fake2'
         self.identity_api.update_project('fake1', tenant)
@@ -465,11 +485,14 @@ class IdentityTests(object):
                           role_id='member')
 
     def test_get_and_remove_role_grant_by_group_and_project(self):
+        new_domain = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex}
+        self.identity_api.create_domain(new_domain['id'], new_domain)
         new_group = {'id': uuid.uuid4().hex, 'domain_id': uuid.uuid4().hex,
                      'name': uuid.uuid4().hex}
         self.identity_api.create_group(new_group['id'], new_group)
         new_user = {'id': uuid.uuid4().hex, 'name': 'new_user',
-                    'password': 'secret', 'enabled': True}
+                    'password': 'secret', 'enabled': True,
+                    'domain_id': new_domain['id']}
         self.identity_api.create_user(new_user['id'], new_user)
         self.identity_api.add_user_to_group(new_user['id'],
                                             new_group['id'])
@@ -505,17 +528,82 @@ class IdentityTests(object):
                      'name': uuid.uuid4().hex}
         self.identity_api.create_group(new_group['id'], new_group)
         new_user = {'id': uuid.uuid4().hex, 'name': 'new_user',
-                    'password': 'secret', 'enabled': True}
+                    'password': uuid.uuid4().hex, 'enabled': True,
+                    'domain_id': new_domain['id']}
         self.identity_api.create_user(new_user['id'], new_user)
         self.identity_api.add_user_to_group(new_user['id'],
                                             new_group['id'])
+
         roles_ref = self.identity_api.list_grants(
             group_id=new_group['id'],
             domain_id=new_domain['id'])
         self.assertEquals(len(roles_ref), 0)
+
         self.identity_api.create_grant(group_id=new_group['id'],
                                        domain_id=new_domain['id'],
                                        role_id='member')
+
+        roles_ref = self.identity_api.list_grants(
+            group_id=new_group['id'],
+            domain_id=new_domain['id'])
+        self.assertDictEqual(roles_ref[0], self.role_member)
+
+        self.identity_api.delete_grant(group_id=new_group['id'],
+                                       domain_id=new_domain['id'],
+                                       role_id='member')
+        roles_ref = self.identity_api.list_grants(
+            group_id=new_group['id'],
+            domain_id=new_domain['id'])
+        self.assertEquals(len(roles_ref), 0)
+        self.assertRaises(exception.NotFound,
+                          self.identity_api.delete_grant,
+                          group_id=new_group['id'],
+                          domain_id=new_domain['id'],
+                          role_id='member')
+
+    def test_get_and_remove_correct_role_grant_from_a_mix(self):
+        new_domain = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex}
+        self.identity_api.create_domain(new_domain['id'], new_domain)
+        new_project = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex,
+                       'domain_id': new_domain['id']}
+        self.identity_api.create_project(new_project['id'], new_project)
+        new_group = {'id': uuid.uuid4().hex, 'domain_id': new_domain['id'],
+                     'name': uuid.uuid4().hex}
+        self.identity_api.create_group(new_group['id'], new_group)
+        new_group2 = {'id': uuid.uuid4().hex, 'domain_id': new_domain['id'],
+                      'name': uuid.uuid4().hex}
+        self.identity_api.create_group(new_group2['id'], new_group2)
+        new_user = {'id': uuid.uuid4().hex, 'name': 'new_user',
+                    'password': uuid.uuid4().hex, 'enabled': True,
+                    'domain_id': new_domain['id']}
+        self.identity_api.create_user(new_user['id'], new_user)
+        new_user2 = {'id': uuid.uuid4().hex, 'name': 'new_user2',
+                     'password': uuid.uuid4().hex, 'enabled': True,
+                     'domain_id': new_domain['id']}
+        self.identity_api.create_user(new_user2['id'], new_user2)
+        self.identity_api.add_user_to_group(new_user['id'],
+                                            new_group['id'])
+        # First check we have no grants
+        roles_ref = self.identity_api.list_grants(
+            group_id=new_group['id'],
+            domain_id=new_domain['id'])
+        self.assertEquals(len(roles_ref), 0)
+        # Now add the grant we are going to test for, and some others as
+        # well just to make sure we get back the right one
+        self.identity_api.create_grant(group_id=new_group['id'],
+                                       domain_id=new_domain['id'],
+                                       role_id='member')
+
+        self.identity_api.create_grant(group_id=new_group2['id'],
+                                       domain_id=new_domain['id'],
+                                       role_id='keystone_admin')
+        self.identity_api.create_grant(user_id=new_user2['id'],
+                                       domain_id=new_domain['id'],
+                                       role_id='keystone_admin')
+        self.identity_api.create_grant(group_id=new_group['id'],
+                                       project_id=new_project['id'],
+                                       role_id='keystone_admin')
+
         roles_ref = self.identity_api.list_grants(
             group_id=new_group['id'],
             domain_id=new_domain['id'])
@@ -538,7 +626,8 @@ class IdentityTests(object):
         new_domain = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex}
         self.identity_api.create_domain(new_domain['id'], new_domain)
         new_user = {'id': uuid.uuid4().hex, 'name': 'new_user',
-                    'password': 'secret', 'enabled': True}
+                    'password': 'secret', 'enabled': True,
+                    'domain_id': new_domain['id']}
         self.identity_api.create_user(new_user['id'], new_user)
         roles_ref = self.identity_api.list_grants(
             user_id=new_user['id'],
@@ -657,6 +746,7 @@ class IdentityTests(object):
     def test_delete_user_with_project_association(self):
         user = {'id': uuid.uuid4().hex,
                 'name': uuid.uuid4().hex,
+                'domain_id': DEFAULT_DOMAIN_ID,
                 'password': uuid.uuid4().hex}
         self.identity_api.create_user(user['id'], user)
         self.identity_api.add_user_to_project(self.tenant_bar['id'],
@@ -669,6 +759,7 @@ class IdentityTests(object):
     def test_delete_user_with_project_roles(self):
         user = {'id': uuid.uuid4().hex,
                 'name': uuid.uuid4().hex,
+                'domain_id': DEFAULT_DOMAIN_ID,
                 'password': uuid.uuid4().hex}
         self.identity_api.create_user(user['id'], user)
         self.identity_api.add_role_to_user_and_project(
@@ -691,33 +782,38 @@ class IdentityTests(object):
                           uuid.uuid4().hex)
 
     def test_create_project_long_name_fails(self):
-        tenant = {'id': 'fake1', 'name': 'a' * 65}
+        tenant = {'id': 'fake1', 'name': 'a' * 65,
+                  'domain_id': DEFAULT_DOMAIN_ID}
         self.assertRaises(exception.ValidationError,
                           self.identity_api.create_project,
                           tenant['id'],
                           tenant)
 
     def test_create_project_blank_name_fails(self):
-        tenant = {'id': 'fake1', 'name': ''}
+        tenant = {'id': 'fake1', 'name': '',
+                  'domain_id': DEFAULT_DOMAIN_ID}
         self.assertRaises(exception.ValidationError,
                           self.identity_api.create_project,
                           tenant['id'],
                           tenant)
 
     def test_create_project_invalid_name_fails(self):
-        tenant = {'id': 'fake1', 'name': None}
+        tenant = {'id': 'fake1', 'name': None,
+                  'domain_id': DEFAULT_DOMAIN_ID}
         self.assertRaises(exception.ValidationError,
                           self.identity_api.create_project,
                           tenant['id'],
                           tenant)
-        tenant = {'id': 'fake1', 'name': 123}
+        tenant = {'id': 'fake1', 'name': 123,
+                  'domain_id': DEFAULT_DOMAIN_ID}
         self.assertRaises(exception.ValidationError,
                           self.identity_api.create_project,
                           tenant['id'],
                           tenant)
 
     def test_update_project_blank_name_fails(self):
-        tenant = {'id': 'fake1', 'name': 'fake1'}
+        tenant = {'id': 'fake1', 'name': 'fake1',
+                  'domain_id': DEFAULT_DOMAIN_ID}
         self.identity_api.create_project('fake1', tenant)
         tenant['name'] = ''
         self.assertRaises(exception.ValidationError,
@@ -726,7 +822,8 @@ class IdentityTests(object):
                           tenant)
 
     def test_update_project_long_name_fails(self):
-        tenant = {'id': 'fake1', 'name': 'fake1'}
+        tenant = {'id': 'fake1', 'name': 'fake1',
+                  'domain_id': DEFAULT_DOMAIN_ID}
         self.identity_api.create_project('fake1', tenant)
         tenant['name'] = 'a' * 65
         self.assertRaises(exception.ValidationError,
@@ -735,7 +832,8 @@ class IdentityTests(object):
                           tenant)
 
     def test_update_project_invalid_name_fails(self):
-        tenant = {'id': 'fake1', 'name': 'fake1'}
+        tenant = {'id': 'fake1', 'name': 'fake1',
+                  'domain_id': DEFAULT_DOMAIN_ID}
         self.identity_api.create_project('fake1', tenant)
         tenant['name'] = None
         self.assertRaises(exception.ValidationError,
@@ -750,34 +848,39 @@ class IdentityTests(object):
                           tenant)
 
     def test_create_user_long_name_fails(self):
-        user = {'id': 'fake1', 'name': 'a' * 65}
+        user = {'id': 'fake1', 'name': 'a' * 65,
+                'domain_id': DEFAULT_DOMAIN_ID}
         self.assertRaises(exception.ValidationError,
                           self.identity_api.create_user,
                           'fake1',
                           user)
 
     def test_create_user_blank_name_fails(self):
-        user = {'id': 'fake1', 'name': ''}
+        user = {'id': 'fake1', 'name': '',
+                'domain_id': DEFAULT_DOMAIN_ID}
         self.assertRaises(exception.ValidationError,
                           self.identity_api.create_user,
                           'fake1',
                           user)
 
     def test_create_user_invalid_name_fails(self):
-        user = {'id': 'fake1', 'name': None}
+        user = {'id': 'fake1', 'name': None,
+                'domain_id': DEFAULT_DOMAIN_ID}
         self.assertRaises(exception.ValidationError,
                           self.identity_api.create_user,
                           'fake1',
                           user)
 
-        user = {'id': 'fake1', 'name': 123}
+        user = {'id': 'fake1', 'name': 123,
+                'domain_id': DEFAULT_DOMAIN_ID}
         self.assertRaises(exception.ValidationError,
                           self.identity_api.create_user,
                           'fake1',
                           user)
 
     def test_update_user_long_name_fails(self):
-        user = {'id': 'fake1', 'name': 'fake1'}
+        user = {'id': 'fake1', 'name': 'fake1',
+                'domain_id': DEFAULT_DOMAIN_ID}
         self.identity_api.create_user('fake1', user)
         user['name'] = 'a' * 65
         self.assertRaises(exception.ValidationError,
@@ -786,7 +889,8 @@ class IdentityTests(object):
                           user)
 
     def test_update_user_blank_name_fails(self):
-        user = {'id': 'fake1', 'name': 'fake1'}
+        user = {'id': 'fake1', 'name': 'fake1',
+                'domain_id': DEFAULT_DOMAIN_ID}
         self.identity_api.create_user('fake1', user)
         user['name'] = ''
         self.assertRaises(exception.ValidationError,
@@ -795,7 +899,8 @@ class IdentityTests(object):
                           user)
 
     def test_update_user_invalid_name_fails(self):
-        user = {'id': 'fake1', 'name': 'fake1'}
+        user = {'id': 'fake1', 'name': 'fake1',
+                'domain_id': DEFAULT_DOMAIN_ID}
         self.identity_api.create_user('fake1', user)
 
         user['name'] = None
@@ -827,8 +932,9 @@ class IdentityTests(object):
                             if x['id'] == test_project['id'])
 
     def test_delete_project_with_role_assignments(self):
-        tenant = {'id': 'fake1', 'name': 'fake1'}
+        tenant = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex,
-        self.identity_api.create_project('fake1', tenant)
+                  'domain_id': DEFAULT_DOMAIN_ID}
+        self.identity_api.create_project(tenant['id'], tenant)
         self.identity_api.add_role_to_user_and_project(
             self.user_foo['id'], tenant['id'], 'member')
         self.identity_api.delete_project(tenant['id'])
@@ -852,20 +958,23 @@ class IdentityTests(object):
         self.assertIn(alt_role['id'], roles_ref)
 
     def test_create_project_doesnt_modify_passed_in_dict(self):
-        new_project = {'id': 'tenant_id', 'name': 'new_project'}
+        new_project = {'id': 'tenant_id', 'name': uuid.uuid4().hex,
+                       'domain_id': DEFAULT_DOMAIN_ID}
         original_project = new_project.copy()
         self.identity_api.create_project('tenant_id', new_project)
         self.assertDictEqual(original_project, new_project)
 
     def test_create_user_doesnt_modify_passed_in_dict(self):
-        new_user = {'id': 'user_id', 'name': 'new_user',
+        new_user = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex,
-                    'password': 'secret', 'enabled': True}
+                    'password': uuid.uuid4().hex, 'enabled': True,
+                    'domain_id': DEFAULT_DOMAIN_ID}
         original_user = new_user.copy()
         self.identity_api.create_user('user_id', new_user)
         self.assertDictEqual(original_user, new_user)
 
     def test_update_user_enable(self):
-        user = {'id': 'fake1', 'name': 'fake1', 'enabled': True}
+        user = {'id': 'fake1', 'name': 'fake1', 'enabled': True,
+                'domain_id': DEFAULT_DOMAIN_ID}
         self.identity_api.create_user('fake1', user)
         user_ref = self.identity_api.get_user('fake1')
         self.assertEqual(user_ref['enabled'], True)
@@ -881,7 +990,8 @@ class IdentityTests(object):
         self.assertEqual(user_ref['enabled'], user['enabled'])
 
     def test_update_project_enable(self):
-        tenant = {'id': 'fake1', 'name': 'fake1', 'enabled': True}
+        tenant = {'id': 'fake1', 'name': 'fake1', 'enabled': True,
+                  'domain_id': DEFAULT_DOMAIN_ID}
         self.identity_api.create_project('fake1', tenant)
         tenant_ref = self.identity_api.get_project('fake1')
         self.assertEqual(tenant_ref['enabled'], True)
@@ -897,11 +1007,14 @@ class IdentityTests(object):
         self.assertEqual(tenant_ref['enabled'], tenant['enabled'])
 
     def test_add_user_to_group(self):
+        domain = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex}
+        self.identity_api.create_domain(domain['id'], domain)
         new_group = {'id': uuid.uuid4().hex, 'domain_id': uuid.uuid4().hex,
                      'name': uuid.uuid4().hex}
         self.identity_api.create_group(new_group['id'], new_group)
         new_user = {'id': uuid.uuid4().hex, 'name': 'new_user',
-                    'password': 'secret', 'enabled': True}
+                    'password': uuid.uuid4().hex, 'enabled': True,
+                    'domain_id': domain['id']}
         self.identity_api.create_user(new_user['id'], new_user)
         self.identity_api.add_user_to_group(new_user['id'],
                                             new_group['id'])
@@ -914,8 +1027,11 @@ class IdentityTests(object):
         self.assertTrue(found)
 
     def test_add_user_to_group_404(self):
+        domain = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex}
+        self.identity_api.create_domain(domain['id'], domain)
         new_user = {'id': uuid.uuid4().hex, 'name': 'new_user',
-                    'password': 'secret', 'enabled': True}
+                    'password': uuid.uuid4().hex, 'enabled': True,
+                    'domain_id': domain['id']}
         self.identity_api.create_user(new_user['id'], new_user)
         self.assertRaises(exception.GroupNotFound,
                           self.identity_api.add_user_to_group,
@@ -931,11 +1047,14 @@ class IdentityTests(object):
                           new_group['id'])
 
     def test_check_user_in_group(self):
+        domain = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex}
+        self.identity_api.create_domain(domain['id'], domain)
         new_group = {'id': uuid.uuid4().hex, 'domain_id': uuid.uuid4().hex,
                      'name': uuid.uuid4().hex}
         self.identity_api.create_group(new_group['id'], new_group)
         new_user = {'id': uuid.uuid4().hex, 'name': 'new_user',
-                    'password': 'secret', 'enabled': True}
+                    'password': uuid.uuid4().hex, 'enabled': True,
+                    'domain_id': domain['id']}
         self.identity_api.create_user(new_user['id'], new_user)
         self.identity_api.add_user_to_group(new_user['id'],
                                             new_group['id'])
@@ -951,11 +1070,14 @@ class IdentityTests(object):
                           new_group['id'])
 
     def test_list_users_in_group(self):
+        domain = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex}
+        self.identity_api.create_domain(domain['id'], domain)
         new_group = {'id': uuid.uuid4().hex, 'domain_id': uuid.uuid4().hex,
                      'name': uuid.uuid4().hex}
         self.identity_api.create_group(new_group['id'], new_group)
         new_user = {'id': uuid.uuid4().hex, 'name': 'new_user',
-                    'password': 'secret', 'enabled': True}
+                    'password': uuid.uuid4().hex, 'enabled': True,
+                    'domain_id': domain['id']}
         self.identity_api.create_user(new_user['id'], new_user)
         self.identity_api.add_user_to_group(new_user['id'],
                                             new_group['id'])
@@ -967,11 +1089,14 @@ class IdentityTests(object):
         self.assertTrue(found)
 
     def test_remove_user_from_group(self):
+        domain = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex}
+        self.identity_api.create_domain(domain['id'], domain)
         new_group = {'id': uuid.uuid4().hex, 'domain_id': uuid.uuid4().hex,
                      'name': uuid.uuid4().hex}
         self.identity_api.create_group(new_group['id'], new_group)
         new_user = {'id': uuid.uuid4().hex, 'name': 'new_user',
-                    'password': 'secret', 'enabled': True}
+                    'password': uuid.uuid4().hex, 'enabled': True,
+                    'domain_id': domain['id']}
         self.identity_api.create_user(new_user['id'], new_user)
         self.identity_api.add_user_to_group(new_user['id'],
                                             new_group['id'])
@@ -983,8 +1108,11 @@ class IdentityTests(object):
             self.assertFalse(x['id'] == new_group['id'])
 
     def test_remove_user_from_group_404(self):
+        domain = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex}
+        self.identity_api.create_domain(domain['id'], domain)
         new_user = {'id': uuid.uuid4().hex, 'name': 'new_user',
-                    'password': 'secret', 'enabled': True}
+                    'password': uuid.uuid4().hex, 'enabled': True,
+                    'domain_id': domain['id']}
         self.identity_api.create_user(new_user['id'], new_user)
         new_group = {'id': uuid.uuid4().hex, 'domain_id': uuid.uuid4().hex,
                      'name': uuid.uuid4().hex}
@@ -1009,20 +1137,52 @@ class IdentityTests(object):
                  'name': uuid.uuid4().hex}
         self.identity_api.create_group(group['id'], group)
         group_ref = self.identity_api.get_group(group['id'])
-        group_ref_dict = dict((x, group_ref[x]) for x in group_ref)
+        self.assertDictEqual(group_ref, group)
-        self.assertDictEqual(group_ref_dict, group)
 
         group['name'] = uuid.uuid4().hex
         self.identity_api.update_group(group['id'], group)
         group_ref = self.identity_api.get_group(group['id'])
-        group_ref_dict = dict((x, group_ref[x]) for x in group_ref)
+        self.assertDictEqual(group_ref, group)
-        self.assertDictEqual(group_ref_dict, group)
 
         self.identity_api.delete_group(group['id'])
         self.assertRaises(exception.GroupNotFound,
                           self.identity_api.get_group,
                           group['id'])
 
+    def test_project_crud(self):
+        project = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex,
+                   'domain_id': uuid.uuid4().hex}
+        self.identity_api.create_project(project['id'], project)
+        project_ref = self.identity_api.get_project(project['id'])
+        self.assertDictEqual(project_ref, project)
+
+        project['name'] = uuid.uuid4().hex
+        self.identity_api.update_project(project['id'], project)
+        project_ref = self.identity_api.get_project(project['id'])
+        self.assertDictEqual(project_ref, project)
+
+        self.identity_api.delete_project(project['id'])
+        self.assertRaises(exception.ProjectNotFound,
+                          self.identity_api.get_project,
+                          project['id'])
+
+    def test_domain_crud(self):
+        domain = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex,
+                  'enabled': True}
+        self.identity_api.create_domain(domain['id'], domain)
+        domain_ref = self.identity_api.get_domain(domain['id'])
+        self.assertDictEqual(domain_ref, domain)
+
+        domain['name'] = uuid.uuid4().hex
+        self.identity_api.update_domain(domain['id'], domain)
+        domain_ref = self.identity_api.get_domain(domain['id'])
+        self.assertDictEqual(domain_ref, domain)
+
+        self.identity_api.delete_domain(domain['id'])
+        self.assertRaises(exception.DomainNotFound,
+                          self.identity_api.get_domain,
+                          domain['id'])
+
 
 class TokenTests(object):
     def test_token_crud(self):

tests/test_backend_ldap.py

@@ -15,6 +15,7 @@
 # under the License.
 
 import uuid
+import nose.exc
 
 from keystone.common.ldap import fakeldap
 from keystone import config
@@ -413,48 +414,57 @@ class LDAPIdentity(test.TestCase, test_backend.IdentityTests):
         user_api.get_connection(user=None, password=None)
 
 # TODO (henry-nash) These need to be removed when the full LDAP implementation
-# is submitted - see BugL #1092187
+# is submitted - see Bugs 1092187, 1101287, 1101276, 1101289
     def test_group_crud(self):
-        pass
+        raise nose.exc.SkipTest('Blocked by bug 1092187')
 
     def test_add_user_to_group(self):
-        pass
+        raise nose.exc.SkipTest('Blocked by bug 1092187')
 
     def test_add_user_to_group_404(self):
-        pass
+        raise nose.exc.SkipTest('Blocked by bug 1092187')
 
     def test_check_user_in_group(self):
-        pass
+        raise nose.exc.SkipTest('Blocked by bug 1092187')
 
     def test_check_user_not_in_group(self):
-        pass
+        raise nose.exc.SkipTest('Blocked by bug 1092187')
 
     def test_list_users_in_group(self):
-        pass
+        raise nose.exc.SkipTest('Blocked by bug 1092187')
 
     def test_remove_user_from_group(self):
-        pass
+        raise nose.exc.SkipTest('Blocked by bug 1092187')
 
     def test_remove_user_from_group_404(self):
-        pass
+        raise nose.exc.SkipTest('Blocked by bug 1092187')
 
     def test_get_role_grant_by_user_and_project(self):
-        pass
+        raise nose.exc.SkipTest('Blocked by bug 1101287')
 
     def test_get_role_grants_for_user_and_project_404(self):
-        pass
+        raise nose.exc.SkipTest('Blocked by bug 1101287')
 
     def test_add_role_grant_to_user_and_project_404(self):
-        pass
+        raise nose.exc.SkipTest('Blocked by bug 1101287')
 
     def test_remove_role_grant_from_user_and_project(self):
-        pass
+        raise nose.exc.SkipTest('Blocked by bug 1101287')
 
     def test_get_and_remove_role_grant_by_group_and_project(self):
-        pass
+        raise nose.exc.SkipTest('Blocked by bug 1101287')
 
     def test_get_and_remove_role_grant_by_group_and_domain(self):
-        pass
+        raise nose.exc.SkipTest('Blocked by bug 1101287')
 
     def test_get_and_remove_role_grant_by_user_and_domain(self):
-        pass
+        raise nose.exc.SkipTest('Blocked by bug 1101287')
+
+    def test_get_and_remove_correct_role_grant_from_a_mix(self):
+        raise nose.exc.SkipTest('Blocked by bug 1101287')
+
+    def test_domain_crud(self):
+        raise nose.exc.SkipTest('Blocked by bug 1101276')
+
+    def test_project_crud(self):
+        raise nose.exc.SkipTest('Blocked by bug 1101289')

tests/test_backend_pam.py

@@ -22,6 +22,7 @@ from keystone import test
 
 
 CONF = config.CONF
+DEFAULT_DOMAIN_ID = CONF.identity.default_domain_id
 
 
 class PamIdentity(test.TestCase):
@@ -41,7 +42,8 @@ class PamIdentity(test.TestCase):
 
     def test_get_project_by_name(self):
         tenant_in_name = self.tenant_in['name']
-        tenant_out = self.identity_api.get_project_by_name(tenant_in_name)
+        tenant_out = self.identity_api.get_project_by_name(
+            tenant_in_name, DEFAULT_DOMAIN_ID)
         self.assertDictEqual(self.tenant_in, tenant_out)
 
     def test_get_user(self):
@@ -49,7 +51,8 @@ class PamIdentity(test.TestCase):
         self.assertDictEqual(self.user_in, user_out)
 
     def test_get_user_by_name(self):
-        user_out = self.identity_api.get_user_by_name(self.user_in['name'])
+        user_out = self.identity_api.get_user_by_name(
+            self.user_in['name'], DEFAULT_DOMAIN_ID)
         self.assertDictEqual(self.user_in, user_out)
 
     def test_get_metadata_for_non_root(self):

tests/test_backend_sql.py

@@ -28,8 +28,8 @@ from keystone import token
 import default_fixtures
 import test_backend
 
-
 CONF = config.CONF
+DEFAULT_DOMAIN_ID = CONF.identity.default_domain_id
 
 
 class SqlTests(test.TestCase):
@@ -65,6 +65,7 @@ class SqlIdentity(SqlTests, test_backend.IdentityTests):
     def test_delete_user_with_project_association(self):
         user = {'id': uuid.uuid4().hex,
                 'name': uuid.uuid4().hex,
+                'domain_id': DEFAULT_DOMAIN_ID,
                 'password': uuid.uuid4().hex}
         self.identity_api.create_user(user['id'], user)
         self.identity_api.add_user_to_project(self.tenant_bar['id'],
@@ -77,6 +78,7 @@ class SqlIdentity(SqlTests, test_backend.IdentityTests):
     def test_create_null_user_name(self):
         user = {'id': uuid.uuid4().hex,
                 'name': None,
+                'domain_id': DEFAULT_DOMAIN_ID,
                 'password': uuid.uuid4().hex}
         self.assertRaises(exception.ValidationError,
                           self.identity_api.create_user,
@@ -87,11 +89,13 @@ class SqlIdentity(SqlTests, test_backend.IdentityTests):
                           user['id'])
         self.assertRaises(exception.UserNotFound,
                           self.identity_api.get_user_by_name,
-                          user['name'])
+                          user['name'],
+                          DEFAULT_DOMAIN_ID)
 
     def test_create_null_project_name(self):
         tenant = {'id': uuid.uuid4().hex,
-                  'name': None}
+                  'name': None,
+                  'domain_id': DEFAULT_DOMAIN_ID}
         self.assertRaises(exception.ValidationError,
                           self.identity_api.create_project,
                           tenant['id'],
@@ -101,7 +105,8 @@ class SqlIdentity(SqlTests, test_backend.IdentityTests):
                           tenant['id'])
         self.assertRaises(exception.ProjectNotFound,
                           self.identity_api.get_project_by_name,
-                          tenant['name'])
+                          tenant['name'],
+                          DEFAULT_DOMAIN_ID)
 
     def test_create_null_role_name(self):
         role = {'id': uuid.uuid4().hex,
@@ -117,6 +122,7 @@ class SqlIdentity(SqlTests, test_backend.IdentityTests):
     def test_delete_project_with_user_association(self):
         user = {'id': 'fake',
                 'name': 'fakeuser',
+                'domain_id': DEFAULT_DOMAIN_ID,
                 'password': 'passwd'}
         self.identity_api.create_user('fake', user)
         self.identity_api.add_user_to_project(self.tenant_bar['id'],
@@ -128,6 +134,7 @@ class SqlIdentity(SqlTests, test_backend.IdentityTests):
     def test_delete_user_with_metadata(self):
         user = {'id': 'fake',
                 'name': 'fakeuser',
+                'domain_id': DEFAULT_DOMAIN_ID,
                 'password': 'passwd'}
         self.identity_api.create_user('fake', user)
         self.identity_api.create_metadata(user['id'],
@@ -142,6 +149,7 @@ class SqlIdentity(SqlTests, test_backend.IdentityTests):
     def test_delete_project_with_metadata(self):
         user = {'id': 'fake',
                 'name': 'fakeuser',
+                'domain_id': DEFAULT_DOMAIN_ID,
                 'password': 'passwd'}
         self.identity_api.create_user('fake', user)
         self.identity_api.create_metadata(user['id'],
@@ -169,6 +177,7 @@ class SqlIdentity(SqlTests, test_backend.IdentityTests):
         tenant = {
             'id': tenant_id,
             'name': uuid.uuid4().hex,
+            'domain_id': DEFAULT_DOMAIN_ID,
             arbitrary_key: arbitrary_value}
         ref = self.identity_api.create_project(tenant_id, tenant)
         self.assertEqual(arbitrary_value, ref[arbitrary_key])
@@ -195,6 +204,7 @@ class SqlIdentity(SqlTests, test_backend.IdentityTests):
         user = {
             'id': user_id,
             'name': uuid.uuid4().hex,
+            'domain_id': DEFAULT_DOMAIN_ID,
             'password': uuid.uuid4().hex,
             arbitrary_key: arbitrary_value}
         ref = self.identity_api.create_user(user_id, user)

tests/test_keystoneclient.py

@@ -22,11 +22,13 @@ import nose.exc
 
 from keystone.openstack.common import jsonutils
 from keystone.openstack.common import timeutils
+from keystone import config
 from keystone import test
 
-
 import default_fixtures
 
+CONF = config.CONF
+DEFAULT_DOMAIN_ID = CONF.identity.default_domain_id
 OPENSTACK_REPO = 'https://review.openstack.org/p/openstack'
 KEYSTONECLIENT_REPO = '%s/python-keystoneclient.git' % OPENSTACK_REPO
 
@@ -862,7 +864,8 @@ class KcMasterTestCase(CompatTestCase, KeystoneClientTests):
         # Add two arbitrary tenants to user for testing purposes
         for i in range(2):
             tenant_id = uuid.uuid4().hex
-            tenant = {'name': 'tenant-%s' % tenant_id, 'id': tenant_id}
+            tenant = {'name': 'tenant-%s' % tenant_id, 'id': tenant_id,
+                      'domain_id': DEFAULT_DOMAIN_ID}
             self.identity_api.create_project(tenant_id, tenant)
             self.identity_api.add_user_to_project(tenant_id,
                                                   self.user_foo['id'])
@@ -888,7 +891,8 @@ class KcMasterTestCase(CompatTestCase, KeystoneClientTests):
         # Add two arbitrary tenants to user for testing purposes
         for i in range(2):
             tenant_id = uuid.uuid4().hex
-            tenant = {'name': 'tenant-%s' % tenant_id, 'id': tenant_id}
+            tenant = {'name': 'tenant-%s' % tenant_id, 'id': tenant_id,
+                      'domain_id': DEFAULT_DOMAIN_ID}
             self.identity_api.create_project(tenant_id, tenant)
             self.identity_api.add_user_to_project(tenant_id,
                                                   self.user_foo['id'])

tests/test_migrate_nova_auth.py

@@ -25,6 +25,7 @@ from keystone import test
 
 
 CONF = config.CONF
+DEFAULT_DOMAIN_ID = CONF.identity.default_domain_id
 
 
 FIXTURE = {
@@ -92,11 +93,13 @@ class MigrateNovaAuth(test.TestCase):
 
         users = {}
         for user in ['user1', 'user2', 'user3', 'user4']:
-            users[user] = self.identity_api.get_user_by_name(user)
+            users[user] = self.identity_api.get_user_by_name(
+                user, DEFAULT_DOMAIN_ID)
 
         tenants = {}
         for tenant in ['proj1', 'proj2', 'proj4']:
-            tenants[tenant] = self.identity_api.get_project_by_name(tenant)
+            tenants[tenant] = self.identity_api.get_project_by_name(
+                tenant, DEFAULT_DOMAIN_ID)
 
         membership_map = {
             'user1': ['proj1'],

tests/test_sql_upgrade.py

@@ -35,12 +35,14 @@ import sqlalchemy
 from keystone.common import sql
 from keystone.common.sql import migration
 from keystone import config
+from keystone import exception
 from keystone import test
 
 import default_fixtures
 
 
 CONF = config.CONF
+DEFAULT_DOMAIN_ID = CONF.identity.default_domain_id
 
 
 class SqlUpgradeTests(test.TestCase):
@@ -129,8 +131,8 @@ class SqlUpgradeTests(test.TestCase):
         self.populate_tenant_table()
         self.upgrade(10)
         self.assertTableColumns("user",
-                                ["id", "name", "extra", "password",
+                                ["id", "name", "extra",
-                                 "enabled"])
+                                 "password", "enabled"])
         self.assertTableColumns("tenant",
                                 ["id", "name", "extra", "description",
                                  "enabled"])
@@ -152,17 +154,33 @@ class SqlUpgradeTests(test.TestCase):
         a_tenant = session.query(tenant_table).filter("id='baz'").one()
         self.assertEqual(a_tenant.description, 'description')
         session.commit()
+        session.close()
 
     def test_downgrade_10_to_8(self):
-        self.upgrade(8)
-        self.populate_user_table()
-        self.populate_tenant_table()
         self.upgrade(10)
+        self.populate_user_table(with_pass_enab=True)
+        self.populate_tenant_table(with_desc_enab=True)
         self.downgrade(8)
+        self.assertTableColumns('user',
+                                ['id', 'name', 'extra'])
+        self.assertTableColumns('tenant',
+                                ['id', 'name', 'extra'])
+        session = self.Session()
+        user_table = sqlalchemy.Table("user",
+                                      self.metadata,
+                                      autoload=True)
+        a_user = session.query(user_table).filter("id='badguy'").one()
+        self.assertEqual(a_user.name, default_fixtures.USERS[2]['name'])
+        tenant_table = sqlalchemy.Table("tenant",
+                                        self.metadata,
+                                        autoload=True)
+        a_tenant = session.query(tenant_table).filter("id='baz'").one()
+        self.assertEqual(a_tenant.name, default_fixtures.TENANTS[1]['name'])
+        session.commit()
+        session.close()
 
     def test_upgrade_10_to_13(self):
         self.upgrade(10)
-
         service_extra = {
             'name': uuid.uuid4().hex,
         }
@@ -187,9 +205,9 @@ class SqlUpgradeTests(test.TestCase):
         self.insert_dict(session, 'service', service)
         self.insert_dict(session, 'endpoint', endpoint)
         session.commit()
+        session.close()
 
         self.upgrade(13)
-
         self.assertTableColumns(
             'service',
             ['id', 'type', 'extra'])
@@ -215,6 +233,8 @@ class SqlUpgradeTests(test.TestCase):
             self.assertEqual(ref.service_id, endpoint['service_id'])
             self.assertEqual(ref.url, endpoint_extra['%surl' % interface])
             self.assertEqual(ref.extra, '{}')
+        session.commit()
+        session.close()
 
     def assertTenantTables(self):
         self.assertTableExists('tenant')
@@ -235,6 +255,12 @@ class SqlUpgradeTests(test.TestCase):
         self.assertProjectTables()
 
     def test_downgrade_project_to_tenant(self):
+        # TODO(henry-nash): Debug why we need to re-load the tenant
+        # or user_tenant_membership ahead of upgrading to project
+        # in order for the assertProjectTables to work on sqlite
+        # (MySQL is fine without it)
+        self.upgrade(14)
+        self.assertTenantTables()
         self.upgrade(15)
         self.assertProjectTables()
         self.downgrade(14)
@@ -248,6 +274,59 @@ class SqlUpgradeTests(test.TestCase):
         self.assertTableExists('group_domain_metadata')
         self.assertTableExists('user_group_membership')
 
+    def test_upgrade_14_to_16(self):
+        self.upgrade(14)
+        self.populate_user_table(with_pass_enab=True)
+        self.populate_tenant_table(with_desc_enab=True)
+        self.upgrade(16)
+        self.assertTableColumns("user",
+                                ["id", "name", "extra",
+                                 "password", "enabled", "domain_id"])
+        session = self.Session()
+        user_table = sqlalchemy.Table("user",
+                                      self.metadata,
+                                      autoload=True)
+        a_user = session.query(user_table).filter("id='foo'").one()
+        self.assertTrue(a_user.enabled)
+        self.assertEqual(a_user.domain_id, DEFAULT_DOMAIN_ID)
+        a_user = session.query(user_table).filter("id='badguy'").one()
+        self.assertEqual(a_user.name, default_fixtures.USERS[2]['name'])
+        self.assertEqual(a_user.domain_id, DEFAULT_DOMAIN_ID)
+        project_table = sqlalchemy.Table("project",
+                                         self.metadata,
+                                         autoload=True)
+        a_project = session.query(project_table).filter("id='baz'").one()
+        self.assertEqual(a_project.description,
+                         default_fixtures.TENANTS[1]['description'])
+        self.assertEqual(a_project.domain_id, DEFAULT_DOMAIN_ID)
+        session.commit()
+        session.close()
+
+    def test_downgrade_16_to_14(self):
+        self.upgrade(16)
+        self.populate_user_table(with_pass_enab_domain=True)
+        self.populate_tenant_table(with_desc_enab_domain=True)
+        self.downgrade(14)
+        self.assertTableColumns("user",
+                                ["id", "name", "extra",
+                                 "password", "enabled"])
+        session = self.Session()
+        user_table = sqlalchemy.Table("user",
+                                      self.metadata,
+                                      autoload=True)
+        a_user = session.query(user_table).filter("id='foo'").one()
+        self.assertTrue(a_user.enabled)
+        a_user = session.query(user_table).filter("id='badguy'").one()
+        self.assertEqual(a_user.name, default_fixtures.USERS[2]['name'])
+        tenant_table = sqlalchemy.Table("tenant",
+                                        self.metadata,
+                                        autoload=True)
+        a_tenant = session.query(tenant_table).filter("id='baz'").one()
+        self.assertEqual(a_tenant.description,
+                         default_fixtures.TENANTS[1]['description'])
+        session.commit()
+        session.close()
+
     def test_downgrade_14_to_13(self):
         self.upgrade(14)
         self.downgrade(13)
@@ -298,6 +377,7 @@ class SqlUpgradeTests(test.TestCase):
             endpoint.update(common_endpoint_attrs)
             self.insert_dict(session, 'endpoint', endpoint)
         session.commit()
+        session.close()
 
         self.downgrade(9)
 
@@ -323,14 +403,15 @@ class SqlUpgradeTests(test.TestCase):
         for interface in ['public', 'internal', 'admin']:
             expected_url = endpoints[interface]['url']
             self.assertEqual(extra['%surl' % interface], expected_url)
+        session.commit()
+        session.close()
 
     def insert_dict(self, session, table_name, d):
         """Naively inserts key-value pairs into a table, given a dictionary."""
-        session.execute(
+        this_table = sqlalchemy.Table(table_name, self.metadata, autoload=True)
-            'INSERT INTO `%s` (%s) VALUES (%s)' % (
+        insert = this_table.insert()
-                table_name,
+        insert.execute(d)
-                ', '.join('%s' % k for k in d.keys()),
+        session.commit()
-                ', '.join("'%s'" % v for v in d.values())))
 
     def test_downgrade_to_0(self):
         self.upgrade(self.max_version)
@@ -355,28 +436,103 @@ class SqlUpgradeTests(test.TestCase):
         self.assertTableColumns('user_domain_metadata',
                                 ['user_id', 'domain_id', 'data'])
 
-    def populate_user_table(self):
+    def populate_user_table(self, with_pass_enab=False,
-        user_table = sqlalchemy.Table('user',
+                            with_pass_enab_domain=False):
+        # Populate the appropriate fields in the user
+        # table, depending on the parameters:
+        #
+        # Default: id, name, extra
+        # pass_enab: Add password, enabled as well
+        # pass_enab_domain: Add password, enabled and domain as well
+        #
+        this_table = sqlalchemy.Table("user",
                                       self.metadata,
                                       autoload=True)
-        session = self.Session()
-        insert = user_table.insert()
         for user in default_fixtures.USERS:
             extra = copy.deepcopy(user)
             extra.pop('id')
             extra.pop('name')
-            user['extra'] = json.dumps(extra)
-            insert.execute(user)
 
-    def populate_tenant_table(self):
+            if with_pass_enab:
+                password = extra.pop('password', None)
+                enabled = extra.pop('enabled', True)
+                ins = this_table.insert().values(
+                    {'id': user['id'],
+                     'name': user['name'],
+                     'password': password,
+                     'enabled': bool(enabled),
+                     'extra': json.dumps(extra)})
+            else:
+                if with_pass_enab_domain:
+                    password = extra.pop('password', None)
+                    enabled = extra.pop('enabled', True)
+                    extra.pop('domain_id')
+                    ins = this_table.insert().values(
+                        {'id': user['id'],
+                         'name': user['name'],
+                         'domain_id': user['domain_id'],
+                         'password': password,
+                         'enabled': bool(enabled),
+                         'extra': json.dumps(extra)})
+                else:
+                    ins = this_table.insert().values(
+                        {'id': user['id'],
+                         'name': user['name'],
+                         'extra': json.dumps(extra)})
+            self.engine.execute(ins)
+
+    def populate_tenant_table(self, with_desc_enab=False,
+                              with_desc_enab_domain=False):
+        # Populate the appropriate fields in the tenant or
+        # project table, depending on the parameters
+        #
+        # Default: id, name, extra
+        # desc_enab: Add description, enabled as well
+        # desc_enab_domain: Add description, enabled and domain as well,
+        #                   plus use project instead of tenant
+        #
+        if with_desc_enab_domain:
+            # By this time tenants are now projects
+            this_table = sqlalchemy.Table("project",
+                                          self.metadata,
+                                          autoload=True)
+        else:
+            this_table = sqlalchemy.Table("tenant",
+                                          self.metadata,
+                                          autoload=True)
+
         for tenant in default_fixtures.TENANTS:
             extra = copy.deepcopy(tenant)
             extra.pop('id')
             extra.pop('name')
-            self.engine.execute("insert into tenant values ('%s', '%s', '%s')"
+
-                                % (tenant['id'],
+            if with_desc_enab:
-                                   tenant['name'],
+                desc = extra.pop('description', None)
-                                   json.dumps(extra)))
+                enabled = extra.pop('enabled', True)
+                ins = this_table.insert().values(
+                    {'id': tenant['id'],
+                     'name': tenant['name'],
+                     'description': desc,
+                     'enabled': bool(enabled),
+                     'extra': json.dumps(extra)})
+            else:
+                if with_desc_enab_domain:
+                    desc = extra.pop('description', None)
+                    enabled = extra.pop('enabled', True)
+                    extra.pop('domain_id')
+                    ins = this_table.insert().values(
+                        {'id': tenant['id'],
+                         'name': tenant['name'],
+                         'domain_id': tenant['domain_id'],
+                         'description': desc,
+                         'enabled': bool(enabled),
+                         'extra': json.dumps(extra)})
+                else:
+                    ins = this_table.insert().values(
+                        {'id': tenant['id'],
+                         'name': tenant['name'],
+                         'extra': json.dumps(extra)})
+            self.engine.execute(ins)
 
     def select_table(self, name):
         table = sqlalchemy.Table(name,
@@ -387,16 +543,21 @@ class SqlUpgradeTests(test.TestCase):
 
     def assertTableExists(self, table_name):
         try:
-            #TODO ayoung: make quoting work for postgres
+            self.select_table(table_name)
-            self.engine.execute("select count(*) from '%s'" % table_name)
+        except sqlalchemy.exc.NoSuchTableError:
-        except:
             raise AssertionError('Table "%s" does not exist' % table_name)
 
     def assertTableDoesNotExist(self, table_name):
         """Asserts that a given table exists cannot be selected by name."""
+        # Switch to a different metadata otherwise you might still
+        # detect renamed or dropped tables
         try:
-            self.assertTableExists(table_name)
+            temp_metadata = sqlalchemy.MetaData()
-        except AssertionError:
+            temp_metadata.bind = self.engine
+            table = sqlalchemy.Table(table_name,
+                                     temp_metadata,
+                                     autoload=True)
+        except sqlalchemy.exc.NoSuchTableError:
             pass
         else:
             raise AssertionError('Table "%s" already exists' % table_name)