Shadow users - Separate user identities

"Shadow users: unified identity" implementation:
Separated user identities from their locally managed credentials by
refactoring the user table into user, local_user, and password tables.

user -> local_user -> password
     -> federated_user
     -> ...

(identity) -> (credentials)

Migrated data from the user table to the local_user and password tables.
Modify backend code to utilize the new tables.

Note: #2 "Shadow LDAP and federated users" will be completed in a
different patch. The federated_user table will be added with that patch.

bp shadow-users

Change-Id: I0b6c188824e856d788fe7156e4a9dc2a04cdb6f8

keystone/common/sql/migrate_repo/versions/090_add_local_user_and_password_tables.py

@@ -0,0 +1,42 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sqlalchemy as sql
+
+
+def upgrade(migrate_engine):
+    meta = sql.MetaData()
+    meta.bind = migrate_engine
+
+    user = sql.Table('user', meta, autoload=True)
+
+    local_user = sql.Table(
+        'local_user',
+        meta,
+        sql.Column('id', sql.Integer, primary_key=True, nullable=False),
+        sql.Column('user_id', sql.String(64),
+                   sql.ForeignKey(user.c.id, ondelete='CASCADE'),
+                   nullable=False, unique=True),
+        sql.Column('domain_id', sql.String(64), nullable=False),
+        sql.Column('name', sql.String(255), nullable=False),
+        sql.UniqueConstraint('domain_id', 'name'))
+    local_user.create(migrate_engine, checkfirst=True)
+
+    password = sql.Table(
+        'password',
+        meta,
+        sql.Column('id', sql.Integer, primary_key=True, nullable=False),
+        sql.Column('local_user_id', sql.Integer,
+                   sql.ForeignKey(local_user.c.id, ondelete='CASCADE'),
+                   nullable=False),
+        sql.Column('password', sql.String(128), nullable=False))
+    password.create(migrate_engine, checkfirst=True)

keystone/common/sql/migrate_repo/versions/091_migrate_data_to_local_user_and_password_tables.py

@@ -0,0 +1,57 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import migrate
+import sqlalchemy as sql
+
+
+def upgrade(migrate_engine):
+    meta = sql.MetaData()
+    meta.bind = migrate_engine
+
+    user_table = sql.Table('user', meta, autoload=True)
+    local_user_table = sql.Table('local_user', meta, autoload=True)
+    password_table = sql.Table('password', meta, autoload=True)
+
+    # migrate data to local_user table
+    local_user_values = []
+    for row in user_table.select().execute():
+        local_user_values.append({'user_id': row['id'],
+                                  'domain_id': row['domain_id'],
+                                  'name': row['name']})
+    if local_user_values:
+        local_user_table.insert().values(local_user_values).execute()
+
+    # migrate data to password table
+    sel = (
+        sql.select([user_table, local_user_table], use_labels=True)
+           .select_from(user_table.join(local_user_table, user_table.c.id ==
+                                        local_user_table.c.user_id))
+    )
+    user_rows = sel.execute()
+    password_values = []
+    for row in user_rows:
+        password_values.append({'local_user_id': row['local_user_id'],
+                                'password': row['user_password']})
+    if password_values:
+        password_table.insert().values(password_values).execute()
+
+    # remove domain_id and name unique constraint
+    if migrate_engine.name != 'sqlite':
+        migrate.UniqueConstraint(user_table.c.domain_id,
+                                 user_table.c.name,
+                                 name='ixu_user_name_domain_id').drop()
+
+    # drop user columns
+    user_table.c.domain_id.drop()
+    user_table.c.name.drop()
+    user_table.c.password.drop()

keystone/identity/backends/sql.py

@@ -12,6 +12,10 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from sqlalchemy import and_
+from sqlalchemy.ext.hybrid import hybrid_property
+from sqlalchemy import orm
+
 from keystone.common import driver_hints
 from keystone.common import sql
 from keystone.common import utils
@@ -25,15 +29,68 @@ class User(sql.ModelBase, sql.DictBase):
     attributes = ['id', 'name', 'domain_id', 'password', 'enabled',
                   'default_project_id']
     id = sql.Column(sql.String(64), primary_key=True)
-    name = sql.Column(sql.String(255), nullable=False)
-    domain_id = sql.Column(sql.String(64), nullable=False)
-    password = sql.Column(sql.String(128))
     enabled = sql.Column(sql.Boolean)
     extra = sql.Column(sql.JsonBlob())
     default_project_id = sql.Column(sql.String(64))
-    # Unique constraint across two columns to create the separation
+    local_user = orm.relationship('LocalUser', uselist=False,
-    # rather than just only 'name' being unique
+                                  single_parent=True,
-    __table_args__ = (sql.UniqueConstraint('domain_id', 'name'),)
+                                  cascade='all,delete-orphan', backref='user')
+
+    # name property
+    @hybrid_property
+    def name(self):
+        if self.local_user:
+            return self.local_user.name
+        else:
+            return None
+
+    @name.setter
+    def name(self, value):
+        if not self.local_user:
+            self.local_user = LocalUser()
+        self.local_user.name = value
+
+    @name.expression
+    def name(cls):
+        return LocalUser.name
+
+    # password property
+    @hybrid_property
+    def password(self):
+        if self.local_user and self.local_user.passwords:
+            return self.local_user.passwords[0].password
+        else:
+            return None
+
+    @password.setter
+    def password(self, value):
+        if not self.local_user:
+            self.local_user = LocalUser()
+        if not self.local_user.passwords:
+            self.local_user.passwords.append(Password())
+        self.local_user.passwords[0].password = value
+
+    @password.expression
+    def password(cls):
+        return Password.password
+
+    # domain_id property
+    @hybrid_property
+    def domain_id(self):
+        if self.local_user:
+            return self.local_user.domain_id
+        else:
+            return None
+
+    @domain_id.setter
+    def domain_id(self, value):
+        if not self.local_user:
+            self.local_user = LocalUser()
+        self.local_user.domain_id = value
+
+    @domain_id.expression
+    def domain_id(cls):
+        return LocalUser.domain_id
 
     def to_dict(self, include_extra_dict=False):
         d = super(User, self).to_dict(include_extra_dict=include_extra_dict)
@@ -42,6 +99,29 @@ class User(sql.ModelBase, sql.DictBase):
         return d
 
 
+class LocalUser(sql.ModelBase, sql.DictBase):
+    __tablename__ = 'local_user'
+    attributes = ['id', 'user_id', 'domain_id', 'name']
+    id = sql.Column(sql.Integer, primary_key=True)
+    user_id = sql.Column(sql.String(64), sql.ForeignKey('user.id',
+                         ondelete='CASCADE'), unique=True)
+    domain_id = sql.Column(sql.String(64), nullable=False)
+    name = sql.Column(sql.String(255), nullable=False)
+    passwords = orm.relationship('Password', single_parent=True,
+                                 cascade='all,delete-orphan',
+                                 backref='local_user')
+    __table_args__ = (sql.UniqueConstraint('domain_id', 'name'), {})
+
+
+class Password(sql.ModelBase, sql.DictBase):
+    __tablename__ = 'password'
+    attributes = ['id', 'local_user_id', 'password']
+    id = sql.Column(sql.Integer, primary_key=True)
+    local_user_id = sql.Column(sql.Integer, sql.ForeignKey('local_user.id',
+                               ondelete='CASCADE'))
+    password = sql.Column(sql.String(128))
+
+
 class Group(sql.ModelBase, sql.DictBase):
     __tablename__ = 'group'
     attributes = ['id', 'name', 'domain_id', 'description']
@@ -118,7 +198,7 @@ class Identity(identity.IdentityDriverV8):
     @driver_hints.truncated
     def list_users(self, hints):
         session = sql.get_session()
-        query = session.query(User)
+        query = session.query(User).outerjoin(LocalUser)
         user_refs = sql.filter_limit_query(User, query, hints)
         return [identity.filter_user(x.to_dict()) for x in user_refs]
 
@@ -134,9 +214,9 @@ class Identity(identity.IdentityDriverV8):
 
     def get_user_by_name(self, user_name, domain_id):
         session = sql.get_session()
-        query = session.query(User)
+        query = session.query(User).join(LocalUser)
-        query = query.filter_by(name=user_name)
+        query = query.filter(and_(LocalUser.name == user_name,
-        query = query.filter_by(domain_id=domain_id)
+                                  LocalUser.domain_id == domain_id))
         try:
             user_ref = query.one()
         except sql.NotFound:
@@ -219,7 +299,8 @@ class Identity(identity.IdentityDriverV8):
     def list_users_in_group(self, group_id, hints):
         session = sql.get_session()
         self.get_group(group_id)
-        query = session.query(User).join(UserGroupMembership)
+        query = session.query(User).outerjoin(LocalUser)
+        query = query.join(UserGroupMembership)
         query = query.filter(UserGroupMembership.group_id == group_id)
         query = sql.filter_limit_query(User, query, hints)
         return [identity.filter_user(u.to_dict()) for u in query]

keystone/tests/unit/test_backend_sql.py

@@ -125,14 +125,24 @@ class SqlModels(SqlTests):
 
     def test_user_model(self):
         cols = (('id', sql.String, 64),
-                ('name', sql.String, 255),
-                ('password', sql.String, 128),
-                ('domain_id', sql.String, 64),
                 ('default_project_id', sql.String, 64),
                 ('enabled', sql.Boolean, None),
                 ('extra', sql.JsonBlob, None))
         self.assertExpectedSchema('user', cols)
 
+    def test_local_user_model(self):
+        cols = (('id', sql.Integer, None),
+                ('user_id', sql.String, 64),
+                ('name', sql.String, 255),
+                ('domain_id', sql.String, 64))
+        self.assertExpectedSchema('local_user', cols)
+
+    def test_password_model(self):
+        cols = (('id', sql.Integer, None),
+                ('local_user_id', sql.Integer, None),
+                ('password', sql.String, 128))
+        self.assertExpectedSchema('password', cols)
+
     def test_group_model(self):
         cols = (('id', sql.String, 64),
                 ('name', sql.String, 64),

keystone/tests/unit/test_sql_upgrade.py

@@ -29,7 +29,6 @@ WARNING::
     all data will be lost.
 """
 
-import copy
 import json
 import uuid
 
@@ -225,6 +224,23 @@ class SqlMigrateBase(unit.SQLDriverOverrides, unit.TestCase):
         else:
             raise AssertionError('Table "%s" already exists' % table_name)
 
+    def assertTableCountsMatch(self, table1_name, table2_name):
+        try:
+            table1 = self.select_table(table1_name)
+        except sqlalchemy.exc.NoSuchTableError:
+            raise AssertionError('Table "%s" does not exist' % table1_name)
+        try:
+            table2 = self.select_table(table2_name)
+        except sqlalchemy.exc.NoSuchTableError:
+            raise AssertionError('Table "%s" does not exist' % table2_name)
+        session = self.Session()
+        table1_count = session.execute(table1.count()).scalar()
+        table2_count = session.execute(table2.count()).scalar()
+        if table1_count != table2_count:
+            raise AssertionError('Table counts do not match: {0} ({1}), {2} '
+                                 '({3})'.format(table1_name, table1_count,
+                                                table2_name, table2_count))
+
     def upgrade(self, *args, **kwargs):
         self._migrate(*args, **kwargs)
 
@@ -700,126 +716,105 @@ class SqlUpgradeTests(SqlMigrateBase):
 
         session.close()
 
-    def populate_user_table(self, with_pass_enab=False,
+    def test_add_local_user_and_password_tables(self):
-                            with_pass_enab_domain=False):
+        local_user_table = 'local_user'
-        # Populate the appropriate fields in the user
+        password_table = 'password'
-        # table, depending on the parameters:
+        self.upgrade(89)
-        #
+        self.assertTableDoesNotExist(local_user_table)
-        # Default: id, name, extra
+        self.assertTableDoesNotExist(password_table)
-        # pass_enab: Add password, enabled as well
+        self.upgrade(90)
-        # pass_enab_domain: Add password, enabled and domain as well
+        self.assertTableColumns(local_user_table,
-        #
+                                ['id',
-        this_table = sqlalchemy.Table("user",
+                                 'user_id',
-                                      self.metadata,
+                                 'domain_id',
-                                      autoload=True)
+                                 'name'])
-        for user in default_fixtures.USERS:
+        self.assertTableColumns(password_table,
-            extra = copy.deepcopy(user)
+                                ['id',
-            extra.pop('id')
+                                 'local_user_id',
-            extra.pop('name')
+                                 'password'])
 
-            if with_pass_enab:
+    def test_migrate_data_to_local_user_and_password_tables(self):
-                password = extra.pop('password', None)
+        def get_expected_users():
-                enabled = extra.pop('enabled', True)
+            expected_users = []
-                ins = this_table.insert().values(
+            for test_user in default_fixtures.USERS:
+                user = {}
+                user['id'] = uuid.uuid4().hex
+                user['name'] = test_user['name']
+                user['domain_id'] = test_user['domain_id']
+                user['password'] = test_user['password']
+                user['enabled'] = True
+                user['extra'] = json.dumps(uuid.uuid4().hex)
+                user['default_project_id'] = uuid.uuid4().hex
+                expected_users.append(user)
+            return expected_users
+
+        def add_users_to_db(expected_users, user_table):
+            for user in expected_users:
+                ins = user_table.insert().values(
                     {'id': user['id'],
                      'name': user['name'],
-                     'password': password,
+                     'domain_id': user['domain_id'],
-                     'enabled': bool(enabled),
+                     'password': user['password'],
-                     'extra': json.dumps(extra)})
+                     'enabled': user['enabled'],
-            else:
+                     'extra': user['extra'],
-                if with_pass_enab_domain:
+                     'default_project_id': user['default_project_id']})
-                    password = extra.pop('password', None)
+                ins.execute()
-                    enabled = extra.pop('enabled', True)
-                    extra.pop('domain_id')
-                    ins = this_table.insert().values(
-                        {'id': user['id'],
-                         'name': user['name'],
-                         'domain_id': user['domain_id'],
-                         'password': password,
-                         'enabled': bool(enabled),
-                         'extra': json.dumps(extra)})
-                else:
-                    ins = this_table.insert().values(
-                        {'id': user['id'],
-                         'name': user['name'],
-                         'extra': json.dumps(extra)})
-            self.engine.execute(ins)
 
-    def populate_tenant_table(self, with_desc_enab=False,
+        def get_users_from_db(user_table, local_user_table, password_table):
-                              with_desc_enab_domain=False):
+            sel = (
-        # Populate the appropriate fields in the tenant or
+                sqlalchemy.select([user_table.c.id,
-        # project table, depending on the parameters
+                                   user_table.c.enabled,
-        #
+                                   user_table.c.extra,
-        # Default: id, name, extra
+                                   user_table.c.default_project_id,
-        # desc_enab: Add description, enabled as well
+                                   local_user_table.c.name,
-        # desc_enab_domain: Add description, enabled and domain as well,
+                                   local_user_table.c.domain_id,
-        #                   plus use project instead of tenant
+                                   password_table.c.password])
-        #
+                .select_from(user_table.join(local_user_table,
-        if with_desc_enab_domain:
+                                             user_table.c.id ==
-            # By this time tenants are now projects
+                                             local_user_table.c.user_id)
-            this_table = sqlalchemy.Table("project",
+                                       .join(password_table,
-                                          self.metadata,
+                                             local_user_table.c.id ==
+                                             password_table.c.local_user_id))
+            )
+            user_rows = sel.execute()
+            users = []
+            for row in user_rows:
+                users.append(
+                    {'id': row['id'],
+                     'name': row['name'],
+                     'domain_id': row['domain_id'],
+                     'password': row['password'],
+                     'enabled': row['enabled'],
+                     'extra': row['extra'],
+                     'default_project_id': row['default_project_id']})
+            return users
+
+        meta = sqlalchemy.MetaData()
+        meta.bind = self.engine
+
+        user_table_name = 'user'
+        local_user_table_name = 'local_user'
+        password_table_name = 'password'
+
+        # populate current user table
+        self.upgrade(90)
+        user_table = sqlalchemy.Table(user_table_name, meta, autoload=True)
+        expected_users = get_expected_users()
+        add_users_to_db(expected_users, user_table)
+
+        # upgrade to migration and test
+        self.upgrade(91)
+        self.assertTableCountsMatch(user_table_name, local_user_table_name)
+        self.assertTableCountsMatch(local_user_table_name, password_table_name)
+        meta.clear()
+        user_table = sqlalchemy.Table(user_table_name, meta, autoload=True)
+        local_user_table = sqlalchemy.Table(local_user_table_name, meta,
+                                            autoload=True)
+        password_table = sqlalchemy.Table(password_table_name, meta,
                                           autoload=True)
-        else:
+        actual_users = get_users_from_db(user_table, local_user_table,
-            this_table = sqlalchemy.Table("tenant",
+                                         password_table)
-                                          self.metadata,
+        self.assertListEqual(expected_users, actual_users)
-                                          autoload=True)
-
-        for tenant in default_fixtures.TENANTS:
-            extra = copy.deepcopy(tenant)
-            extra.pop('id')
-            extra.pop('name')
-
-            if with_desc_enab:
-                desc = extra.pop('description', None)
-                enabled = extra.pop('enabled', True)
-                ins = this_table.insert().values(
-                    {'id': tenant['id'],
-                     'name': tenant['name'],
-                     'description': desc,
-                     'enabled': bool(enabled),
-                     'extra': json.dumps(extra)})
-            else:
-                if with_desc_enab_domain:
-                    desc = extra.pop('description', None)
-                    enabled = extra.pop('enabled', True)
-                    extra.pop('domain_id')
-                    ins = this_table.insert().values(
-                        {'id': tenant['id'],
-                         'name': tenant['name'],
-                         'domain_id': tenant['domain_id'],
-                         'description': desc,
-                         'enabled': bool(enabled),
-                         'extra': json.dumps(extra)})
-                else:
-                    ins = this_table.insert().values(
-                        {'id': tenant['id'],
-                         'name': tenant['name'],
-                         'extra': json.dumps(extra)})
-            self.engine.execute(ins)
-
-    def _mysql_check_all_tables_innodb(self):
-        database = self.engine.url.database
-
-        connection = self.engine.connect()
-        # sanity check
-        total = connection.execute("SELECT count(*) "
-                                   "from information_schema.TABLES "
-                                   "where TABLE_SCHEMA='%(database)s'" %
-                                   dict(database=database))
-        self.assertTrue(total.scalar() > 0, "No tables found. Wrong schema?")
-
-        noninnodb = connection.execute("SELECT table_name "
-                                       "from information_schema.TABLES "
-                                       "where TABLE_SCHEMA='%(database)s' "
-                                       "and ENGINE!='InnoDB' "
-                                       "and TABLE_NAME!='migrate_version'" %
-                                       dict(database=database))
-        names = [x[0] for x in noninnodb]
-        self.assertEqual([], names,
-                         "Non-InnoDB tables exist")
-
-        connection.close()
 
 
 class VersionTests(SqlMigrateBase):