diff --git a/keystone/backends.py b/keystone/backends.py index 8380cdfcca..91709618da 100644 --- a/keystone/backends.py +++ b/keystone/backends.py @@ -37,6 +37,8 @@ def load_backends(): catalog_api=catalog.Manager(), credential_api=credential.Manager(), endpoint_filter_api=endpoint_filter.Manager(), + id_generator_api=identity.generator.Manager(), + id_mapping_api=identity.MappingManager(), identity_api=_IDENTITY_API, policy_api=policy.Manager(), token_api=token.Manager(), diff --git a/keystone/common/sql/migrate_repo/versions/051_add_id_mapping.py b/keystone/common/sql/migrate_repo/versions/051_add_id_mapping.py new file mode 100644 index 0000000000..074fbb6326 --- /dev/null +++ b/keystone/common/sql/migrate_repo/versions/051_add_id_mapping.py @@ -0,0 +1,49 @@ +# Copyright 2014 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sql + +from keystone.identity.mapping_backends import mapping + + +MAPPING_TABLE = 'id_mapping' + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + mapping_table = sql.Table( + MAPPING_TABLE, + meta, + sql.Column('public_id', sql.String(64), primary_key=True), + sql.Column('domain_id', sql.String(64), nullable=False), + sql.Column('local_id', sql.String(64), nullable=False), + sql.Column('entity_type', sql.Enum( + mapping.EntityType.USER, + mapping.EntityType.GROUP, + name='entity_type'), + nullable=False), + sql.UniqueConstraint('domain_id', 'local_id', 'entity_type'), + mysql_engine='InnoDB', + mysql_charset='utf8') + mapping_table.create(migrate_engine, checkfirst=True) + + +def downgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + assignment = sql.Table(MAPPING_TABLE, meta, autoload=True) + assignment.drop(migrate_engine, checkfirst=True) diff --git a/keystone/identity/__init__.py b/keystone/identity/__init__.py index 4e4d16c50a..3063b5ca02 100644 --- a/keystone/identity/__init__.py +++ b/keystone/identity/__init__.py @@ -14,4 +14,5 @@ from keystone.identity import controllers # noqa from keystone.identity.core import * # noqa +from keystone.identity import generator # noqa from keystone.identity import routers # noqa diff --git a/keystone/identity/core.py b/keystone/identity/core.py index 8a886e50df..9d7efbbf46 100644 --- a/keystone/identity/core.py +++ b/keystone/identity/core.py @@ -682,3 +682,75 @@ class Driver(object): raise exception.NotImplemented() # end of identity + + +@dependency.provider('id_mapping_api') +class MappingManager(manager.Manager): + """Default pivot point for the ID Mapping backend.""" + + def __init__(self): + # TODO(henry-nash): Use a config option to select the mapping driver + super(MappingManager, self).__init__( + 'keystone.identity.mapping_backends.sql.Mapping') + + +@six.add_metaclass(abc.ABCMeta) +class MappingDriver(object): + """Interface description for an ID Mapping driver.""" + + @abc.abstractmethod + def get_public_id(self, local_entity): + """Returns the public ID for the given local entity. + + :param dict local_entity: Containing the entity domain, local ID and + type ('user' or 'group'). + :returns: public ID, or None if no mapping is found. + + """ + raise exception.NotImplemented() + + @abc.abstractmethod + def get_id_mapping(self, public_id): + """Returns the local mapping. + + :param public_id: The public ID for the mapping required. + :returns dict: Containing the entity domain, local ID and type. If no + mapping is found, it returns None. + + """ + raise exception.NotImplemented() + + @abc.abstractmethod + def create_id_mapping(self, local_entity, public_id=None): + """Create and store a mapping to a public_id. + + :param dict local_entity: Containing the entity domain, local ID and + type ('user' or 'group'). + :param public_id: If specified, this will be the public ID. If this + is not specified, a public ID will be generated. + :returns: public ID + + """ + raise exception.NotImplemented() + + @abc.abstractmethod + def delete_id_mapping(self, public_id): + """Deletes an entry for the given public_id. + + :param public_id: The public ID for the mapping to be deleted. + + The method is silent if no mapping is found. + + """ + raise exception.NotImplemented() + + @abc.abstractmethod + def purge_mappings(self, purge_filter): + """Purge selected identity mappings. + + :param dict purge_filter: Containing the attributes of the filter that + defines which entries to purge. An empty + filter means purge all mappings. + + """ + raise exception.NotImplemented() diff --git a/keystone/identity/generator.py b/keystone/identity/generator.py new file mode 100644 index 0000000000..e034dd3f6d --- /dev/null +++ b/keystone/identity/generator.py @@ -0,0 +1,51 @@ +# Copyright 2014 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""ID Generator provider interface.""" + +import abc + +import six + +from keystone.common import dependency +from keystone.common import manager +from keystone import exception + + +@dependency.provider('id_generator_api') +class Manager(manager.Manager): + """Default pivot point for the identifier generator backend.""" + + def __init__(self): + # TODO(henry-nash): Use a config option to select the generator driver + super(Manager, self).__init__( + 'keystone.identity.id_generators.sha256.Generator') + + +@six.add_metaclass(abc.ABCMeta) +class IDGenerator(object): + """Interface description for an ID Generator provider.""" + + @abc.abstractmethod + def generate_public_ID(self, mapping): + """Return a Public ID for the given mapping dict. + + :param dict mapping: The items to be hashed. + + The ID must be reproduceable and no more than 64 chars in length. + The ID generated should be independant of the order of the items + in the mapping dict. + + """ + raise exception.NotImplemented() diff --git a/keystone/identity/id_generators/__init__.py b/keystone/identity/id_generators/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/keystone/identity/id_generators/sha256.py b/keystone/identity/id_generators/sha256.py new file mode 100644 index 0000000000..89c2c14b43 --- /dev/null +++ b/keystone/identity/id_generators/sha256.py @@ -0,0 +1,28 @@ +# Copyright 2014 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import hashlib + +import six + +from keystone.identity import generator + + +class Generator(generator.IDGenerator): + + def generate_public_ID(self, mapping): + m = hashlib.sha256() + for key in sorted(six.iterkeys(mapping)): + m.update(mapping[key]) + return m.hexdigest() diff --git a/keystone/identity/mapping_backends/__init__.py b/keystone/identity/mapping_backends/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/keystone/identity/mapping_backends/mapping.py b/keystone/identity/mapping_backends/mapping.py new file mode 100644 index 0000000000..ca738188ce --- /dev/null +++ b/keystone/identity/mapping_backends/mapping.py @@ -0,0 +1,18 @@ +# Copyright 2014 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class EntityType: + USER = 'user' + GROUP = 'group' diff --git a/keystone/identity/mapping_backends/sql.py b/keystone/identity/mapping_backends/sql.py new file mode 100644 index 0000000000..45a44fb8cf --- /dev/null +++ b/keystone/identity/mapping_backends/sql.py @@ -0,0 +1,93 @@ +# Copyright 2014 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from keystone.common import dependency +from keystone.common import sql +from keystone import identity +from keystone.identity.mapping_backends import mapping as identity_mapping + + +class IDMapping(sql.ModelBase, sql.ModelDictMixin): + __tablename__ = 'id_mapping' + public_id = sql.Column(sql.String(64), primary_key=True) + domain_id = sql.Column(sql.String(64), nullable=False) + local_id = sql.Column(sql.String(64), nullable=False) + # NOTE(henry-nash); Postgres requires a name to be defined for an Enum + entity_type = sql.Column( + sql.Enum(identity_mapping.EntityType.USER, + identity_mapping.EntityType.GROUP, + name='entity_type'), + nullable=False) + # Unique constraint to ensure you can't store more than one mapping to the + # same underlying values + __table_args__ = ( + sql.UniqueConstraint('domain_id', 'local_id', 'entity_type'), {}) + + +@dependency.requires('id_generator_api') +class Mapping(identity.MappingDriver): + + def get_public_id(self, local_entity): + # NOTE(henry-nash): Since the Public ID is regeneratable, rather + # than search for the entry using the local entity values, we + # could create the hash and do a PK lookup. However this would only + # work if we hashed all the entries, even those that already generate + # UUIDs, like SQL. Further, this would only work if the generation + # algorithm was immutable (e.g. it had always been sha256). + session = sql.get_session() + query = session.query(IDMapping.public_id) + query = query.filter_by(domain_id=local_entity['domain_id']) + query = query.filter_by(local_id=local_entity['local_id']) + query = query.filter_by(entity_type=local_entity['entity_type']) + try: + public_ref = query.one() + public_id = public_ref.public_id + return public_id + except sql.NotFound: + return None + + def get_id_mapping(self, public_id): + session = sql.get_session() + mapping_ref = session.query(IDMapping).get(public_id) + if mapping_ref: + return mapping_ref.to_dict() + + def create_id_mapping(self, local_entity, public_id=None): + entity = local_entity.copy() + with sql.transaction() as session: + if public_id is None: + public_id = self.id_generator_api.generate_public_ID(entity) + entity['public_id'] = public_id + mapping_ref = IDMapping.from_dict(entity) + session.add(mapping_ref) + return public_id + + def delete_id_mapping(self, public_id): + with sql.transaction() as session: + ref = session.query(IDMapping).get(public_id) + if ref: + session.delete(ref) + + def purge_mappings(self, purge_filter): + session = sql.get_session() + query = session.query(IDMapping) + if 'domain_id' in purge_filter: + query = query.filter_by(domain_id=purge_filter['domain_id']) + if 'public_id' in purge_filter: + query = query.filter_by(public_id=purge_filter['public_id']) + if 'local_id' in purge_filter: + query = query.filter_by(local_id=purge_filter['local_id']) + if 'entity_type' in purge_filter: + query = query.filter_by(entity_type=purge_filter['entity_type']) + query.delete() diff --git a/keystone/tests/identity_mapping.py b/keystone/tests/identity_mapping.py new file mode 100644 index 0000000000..bb02619445 --- /dev/null +++ b/keystone/tests/identity_mapping.py @@ -0,0 +1,30 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from keystone.common import sql +from keystone.identity.mapping_backends import sql as mapping_sql + +# NOTE(henry-nash): This function is defined in a separate file since it will +# be used across multiple unit test files once the full support for cross +# backend identifiers is implemented. +# +# TODO(henry-nash): Remove this comment once the full support mentioned above +# has landed, since the reason for this separate file will be obvious. + + +def list_id_mappings(): + """List all id_mappings for testing purposes.""" + + a_session = sql.get_session() + refs = a_session.query(mapping_sql.IDMapping).all() + return [x.to_dict() for x in refs] diff --git a/keystone/tests/test_backend_id_mapping_sql.py b/keystone/tests/test_backend_id_mapping_sql.py new file mode 100644 index 0000000000..b8cdff1416 --- /dev/null +++ b/keystone/tests/test_backend_id_mapping_sql.py @@ -0,0 +1,181 @@ +# Copyright 2014 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import uuid + +from testtools import matchers + +from keystone.common import sql +from keystone.identity.mapping_backends import mapping +from keystone.tests import identity_mapping as mapping_sql +from keystone.tests import test_backend_sql + + +class SqlIDMappingTable(test_backend_sql.SqlModels): + """Set of tests for checking SQL Identity ID Mapping.""" + + def test_id_mapping(self): + cols = (('public_id', sql.String, 64), + ('domain_id', sql.String, 64), + ('local_id', sql.String, 64), + ('entity_type', sql.Enum, None)) + self.assertExpectedSchema('id_mapping', cols) + + +class SqlIDMapping(test_backend_sql.SqlTests): + + def setUp(self): + super(SqlIDMapping, self).setUp() + self.load_sample_data() + + def load_sample_data(self): + self.addCleanup(self.clean_sample_data) + domainA = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex} + self.domainA = self.assignment_api.create_domain(domainA['id'], + domainA) + domainB = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex} + self.domainB = self.assignment_api.create_domain(domainB['id'], + domainB) + + def clean_sample_data(self): + if hasattr(self, 'domainA'): + self.domainA['enabled'] = False + self.assignment_api.update_domain(self.domainA['id'], self.domainA) + self.assignment_api.delete_domain(self.domainA['id']) + if hasattr(self, 'domainB'): + self.domainB['enabled'] = False + self.assignment_api.update_domain(self.domainB['id'], self.domainB) + self.assignment_api.delete_domain(self.domainB['id']) + + def test_invalid_public_key(self): + self.assertIsNone(self.id_mapping_api.get_id_mapping(uuid.uuid4().hex)) + + def test_id_mapping_crud(self): + initial_mappings = len(mapping_sql.list_id_mappings()) + local_id1 = uuid.uuid4().hex + local_id2 = uuid.uuid4().hex + local_entity1 = {'domain_id': self.domainA['id'], + 'local_id': local_id1, + 'entity_type': mapping.EntityType.USER} + local_entity2 = {'domain_id': self.domainB['id'], + 'local_id': local_id2, + 'entity_type': mapping.EntityType.GROUP} + + # Check no mappings for the new local entities + self.assertIsNone(self.id_mapping_api.get_public_id(local_entity1)) + self.assertIsNone(self.id_mapping_api.get_public_id(local_entity2)) + + # Create the new mappings and then read them back + public_id1 = self.id_mapping_api.create_id_mapping(local_entity1) + public_id2 = self.id_mapping_api.create_id_mapping(local_entity2) + self.assertThat(mapping_sql.list_id_mappings(), + matchers.HasLength(initial_mappings + 2)) + self.assertEqual( + public_id1, self.id_mapping_api.get_public_id(local_entity1)) + self.assertEqual( + public_id2, self.id_mapping_api.get_public_id(local_entity2)) + + local_id_ref = self.id_mapping_api.get_id_mapping(public_id1) + self.assertEqual(self.domainA['id'], local_id_ref['domain_id']) + self.assertEqual(local_id1, local_id_ref['local_id']) + self.assertEqual(mapping.EntityType.USER, local_id_ref['entity_type']) + # Check we have really created a new external ID + self.assertNotEqual(local_id1, public_id1) + + local_id_ref = self.id_mapping_api.get_id_mapping(public_id2) + self.assertEqual(self.domainB['id'], local_id_ref['domain_id']) + self.assertEqual(local_id2, local_id_ref['local_id']) + self.assertEqual(mapping.EntityType.GROUP, local_id_ref['entity_type']) + # Check we have really created a new external ID + self.assertNotEqual(local_id2, public_id2) + + # Create another mappings, this time specifying a public ID to use + new_public_id = uuid.uuid4().hex + public_id3 = self.id_mapping_api.create_id_mapping( + {'domain_id': self.domainB['id'], 'local_id': local_id2, + 'entity_type': mapping.EntityType.USER}, + public_id=new_public_id) + self.assertEqual(new_public_id, public_id3) + self.assertThat(mapping_sql.list_id_mappings(), + matchers.HasLength(initial_mappings + 3)) + + # Delete the mappings we created, and make sure the mapping count + # goes back to where it was + self.id_mapping_api.delete_id_mapping(public_id1) + self.id_mapping_api.delete_id_mapping(public_id2) + self.id_mapping_api.delete_id_mapping(public_id3) + self.assertThat(mapping_sql.list_id_mappings(), + matchers.HasLength(initial_mappings)) + + def test_delete_public_id_is_silent(self): + # Test that deleting an invalid public key is silent + self.id_mapping_api.delete_id_mapping(uuid.uuid4().hex) + + def test_purge_mappings(self): + initial_mappings = len(mapping_sql.list_id_mappings()) + local_id1 = uuid.uuid4().hex + local_id2 = uuid.uuid4().hex + local_id3 = uuid.uuid4().hex + local_id4 = uuid.uuid4().hex + local_id5 = uuid.uuid4().hex + + # Create five mappings,two in domainA, three in domainB + self.id_mapping_api.create_id_mapping( + {'domain_id': self.domainA['id'], 'local_id': local_id1, + 'entity_type': mapping.EntityType.USER}) + self.id_mapping_api.create_id_mapping( + {'domain_id': self.domainA['id'], 'local_id': local_id2, + 'entity_type': mapping.EntityType.USER}) + public_id3 = self.id_mapping_api.create_id_mapping( + {'domain_id': self.domainB['id'], 'local_id': local_id3, + 'entity_type': mapping.EntityType.GROUP}) + public_id4 = self.id_mapping_api.create_id_mapping( + {'domain_id': self.domainB['id'], 'local_id': local_id4, + 'entity_type': mapping.EntityType.USER}) + public_id5 = self.id_mapping_api.create_id_mapping( + {'domain_id': self.domainB['id'], 'local_id': local_id5, + 'entity_type': mapping.EntityType.USER}) + + self.assertThat(mapping_sql.list_id_mappings(), + matchers.HasLength(initial_mappings + 5)) + + # Purge mappings for domainA, should be left with those in B + self.id_mapping_api.purge_mappings( + {'domain_id': self.domainA['id']}) + self.assertThat(mapping_sql.list_id_mappings(), + matchers.HasLength(initial_mappings + 3)) + self.id_mapping_api.get_id_mapping(public_id3) + self.id_mapping_api.get_id_mapping(public_id4) + self.id_mapping_api.get_id_mapping(public_id5) + + # Purge mappings for type Group, should purge one more + self.id_mapping_api.purge_mappings( + {'entity_type': mapping.EntityType.GROUP}) + self.assertThat(mapping_sql.list_id_mappings(), + matchers.HasLength(initial_mappings + 2)) + self.id_mapping_api.get_id_mapping(public_id4) + self.id_mapping_api.get_id_mapping(public_id5) + + # Purge mapping for a specific local identifier + self.id_mapping_api.purge_mappings( + {'domain_id': self.domainB['id'], 'local_id': local_id4, + 'entity_type': mapping.EntityType.USER}) + self.assertThat(mapping_sql.list_id_mappings(), + matchers.HasLength(initial_mappings + 1)) + self.id_mapping_api.get_id_mapping(public_id5) + + # Purge mappings the remaining mappings + self.id_mapping_api.purge_mappings({}) + self.assertThat(mapping_sql.list_id_mappings(), + matchers.HasLength(initial_mappings)) diff --git a/keystone/tests/test_sql_upgrade.py b/keystone/tests/test_sql_upgrade.py index 73598a6cc0..045c431b23 100644 --- a/keystone/tests/test_sql_upgrade.py +++ b/keystone/tests/test_sql_upgrade.py @@ -1203,6 +1203,14 @@ class SqlUpgradeTests(SqlMigrateBase): add_region(region_nonunique) self.assertEqual(2, session.query(region_nonunique).count()) + def test_id_mapping(self): + self.upgrade(50) + self.assertTableDoesNotExist('id_mapping') + self.upgrade(51) + self.assertTableExists('id_mapping') + self.downgrade(50) + self.assertTableDoesNotExist('id_mapping') + def populate_user_table(self, with_pass_enab=False, with_pass_enab_domain=False): # Populate the appropriate fields in the user