From b1fdad9875542282abd630a6cdeed6686b0e5beb Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Fri, 15 Jul 2016 20:57:45 +0300 Subject: [PATCH] Add mapping_populate command Fetching users from LDAP requires creating public ids for them. id_mapping_api does that. Creating public ids is slow, because it requires performing N INSERTs for N users, and there is no way to work around that. It leads to very slow responses to queries like "list users". By pre-creating these public ids we improve API users' experience. Add keystone-manage mapping_populate command that creates id mapping entries for users. bp ldap-preprocessing Partial-Bug: 1582585 Change-Id: I98f795854aee26f9e7f668372c47572d2b6d4f0f --- doc/source/configuration.rst | 10 +++ doc/source/man/keystone-manage.rst | 1 + keystone/cmd/cli.py | 47 ++++++++++++++ keystone/tests/unit/test_cli.py | 64 +++++++++++++++++++ .../mapping_populate-521d92445505b8a3.yaml | 13 ++++ 5 files changed, 135 insertions(+) create mode 100644 releasenotes/notes/mapping_populate-521d92445505b8a3.yaml diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst index ef011dda3e..52fe0d9372 100644 --- a/doc/source/configuration.rst +++ b/doc/source/configuration.rst @@ -280,6 +280,15 @@ with the following command: $ keystone-manage mapping_purge --all +Generating public IDs in the first run may take a while, and most probably +first API requests to fetch user list will fail by timeout. To prevent this, +``mapping_populate`` command should be executed. It should be executed right after +LDAP has been configured or after ``mapping_purge``. + +.. code-block:: bash + + $ keystone-manage mapping_populate --domain DOMAINA + Public ID Generators -------------------- @@ -1308,6 +1317,7 @@ through the normal REST API. At the moment, the following calls are supported: * ``fernet_rotate``: Rotate keys in the Fernet key repository. * ``fernet_setup``: Setup a Fernet key repository. * ``mapping_engine``: Test your federation mapping rules. +* ``mapping_populate``: Prepare domain-specific LDAP backend * ``mapping_purge``: Purge the identity mapping table. * ``pki_setup``: Initialize the certificates used to sign tokens. * ``saml_idp_metadata``: Generate identity provider metadata. diff --git a/doc/source/man/keystone-manage.rst b/doc/source/man/keystone-manage.rst index 2ce9a9c72e..2855c0e316 100644 --- a/doc/source/man/keystone-manage.rst +++ b/doc/source/man/keystone-manage.rst @@ -47,6 +47,7 @@ Available commands: * ``domain_config_upload``: Upload domain configuration file. * ``fernet_rotate``: Rotate keys in the Fernet key repository. * ``fernet_setup``: Setup a Fernet key repository. +* ``mapping_populate``: Prepare domain-specific LDAP backend. * ``mapping_purge``: Purge the identity mapping table. * ``mapping_engine``: Test your federation mapping rules. * ``pki_setup``: Initialize the certificates used to sign tokens. **deprecated** diff --git a/keystone/cmd/cli.py b/keystone/cmd/cli.py index 6724c90db4..44d9fb199c 100644 --- a/keystone/cmd/cli.py +++ b/keystone/cmd/cli.py @@ -1004,6 +1004,52 @@ class MappingEngineTester(BaseApp): "engine.")) +class MappingPopulate(BaseApp): + """Pre-populate entries from domain-specific backends. + + Running this command is not required. It should only be run right after + the LDAP was configured, when many new users were added, or when + "mapping_purge" is run. + + This command will take a while to run. It is perfectly fine for it to run + more than several minutes. + """ + + name = "mapping_populate" + + @classmethod + def load_backends(cls): + drivers = backends.load_backends() + cls.identity_api = drivers['identity_api'] + cls.resource_api = drivers['resource_api'] + + @classmethod + def add_argument_parser(cls, subparsers): + parser = super(MappingPopulate, cls).add_argument_parser( + subparsers) + + parser.add_argument('--domain-name', default=None, required=True, + help=("Name of the domain configured to use" + "domain-specific backend")) + return parser + + @classmethod + def main(cls): + """Process entries for id_mapping_api.""" + cls.load_backends() + domain_name = CONF.command.domain_name + try: + domain_id = cls.resource_api.get_domain_by_name(domain_name)['id'] + except exception.DomainNotFound: + print(_('Invalid domain name or ID: %(domain)s') % { + 'domain': domain_id}) + return False + # We don't actually need to tackle id_mapping_api in order to get + # entries there, because list_users does this anyway. That's why it + # will be enough to just make the call below. + cls.identity_api.list_users(domain_scope=domain_id) + + CMDS = [ BootStrap, DbSync, @@ -1012,6 +1058,7 @@ CMDS = [ DomainConfigUpload, FernetRotate, FernetSetup, + MappingPopulate, MappingPurge, MappingEngineTester, PKISetup, diff --git a/keystone/tests/unit/test_cli.py b/keystone/tests/unit/test_cli.py index f412fbd39b..4852f5b07a 100644 --- a/keystone/tests/unit/test_cli.py +++ b/keystone/tests/unit/test_cli.py @@ -28,8 +28,11 @@ from keystone.common import dependency from keystone.common.sql import migration_helpers import keystone.conf from keystone.i18n import _ +from keystone.identity.mapping_backends import mapping as identity_mapping from keystone.tests import unit +from keystone.tests.unit import default_fixtures from keystone.tests.unit.ksfixtures import database +from keystone.tests.unit.ksfixtures import ldapdb CONF = keystone.conf.CONF @@ -596,3 +599,64 @@ class CliDBSyncTestCase(unit.BaseTestCase): CONF, 'command', self.FakeConfCommand(self))) cli.DbSync.main() self._assert_correct_call(migration_helpers.contract_schema) + + +class TestMappingPopulate(unit.SQLDriverOverrides, unit.TestCase): + + def setUp(self): + sqldb = self.useFixture(database.Database()) + super(TestMappingPopulate, self).setUp() + self.ldapdb = self.useFixture(ldapdb.LDAPDatabase()) + self.ldapdb.clear() + + self.load_backends() + + sqldb.recreate() + self.load_fixtures(default_fixtures) + + def config_files(self): + self.config_fixture.register_cli_opt(cli.command_opt) + config_files = super(TestMappingPopulate, self).config_files() + config_files.append(unit.dirs.tests_conf('backend_ldap_sql.conf')) + return config_files + + def config_overrides(self): + super(TestMappingPopulate, self).config_overrides() + self.config_fixture.config(group='identity', driver='ldap') + self.config_fixture.config(group='identity_mapping', + backward_compatible_ids=False) + + def config(self, config_files): + CONF(args=['mapping_populate', '--domain-name', 'Default'], + project='keystone', + default_config_files=config_files) + + def test_mapping_populate(self): + # mapping_populate should create id mappings. Test plan: + # 0. Purge mappings + # 1. Fetch user list directly via backend. It will not create any + # mappings because it bypasses identity manager + # 2. Verify that users have no public_id yet + # 3. Execute mapping_populate. It should create id mappings + # 4. For the same users verify that they have public_id now + purge_filter = {} + self.id_mapping_api.purge_mappings(purge_filter) + hints = None + users = self.identity_api.driver.list_users(hints) + for user in users: + local_entity = { + 'domain_id': CONF.identity.default_domain_id, + 'local_id': user['id'], + 'entity_type': identity_mapping.EntityType.USER} + self.assertIsNone(self.id_mapping_api.get_public_id(local_entity)) + + dependency.reset() # backends are loaded again in the command handler + cli.MappingPopulate.main() + + for user in users: + local_entity = { + 'domain_id': CONF.identity.default_domain_id, + 'local_id': user['id'], + 'entity_type': identity_mapping.EntityType.USER} + self.assertIsNotNone( + self.id_mapping_api.get_public_id(local_entity)) diff --git a/releasenotes/notes/mapping_populate-521d92445505b8a3.yaml b/releasenotes/notes/mapping_populate-521d92445505b8a3.yaml new file mode 100644 index 0000000000..a1b42a64c6 --- /dev/null +++ b/releasenotes/notes/mapping_populate-521d92445505b8a3.yaml @@ -0,0 +1,13 @@ +--- +prelude: > + Add ``keystone-manage mapping_populate`` command, which + should be used when domain-specific LDAP backend is + used. +features: + - Add ``keystone-manage mapping_populate`` command. + This command will pre-populate a mapping table with all + users from LDAP, in order to improve future query + performance. It should be used when an LDAP is first + configured, or after calling ``keystone-manage mapping_purge``, + before any queries related to the domain are made. For more + information see ``keystone-manage mapping_populate --help``