Add mapping_populate command

Fetching users from LDAP requires creating public ids for them.
id_mapping_api does that. Creating public ids is slow, because it
requires performing N INSERTs for N users, and there is no way to
work around that. It leads to very slow responses to queries like
"list users".

By pre-creating these public ids we improve API users' experience.

Add keystone-manage mapping_populate command that creates id mapping entries
for users.

bp ldap-preprocessing
Partial-Bug: 1582585
Change-Id: I98f795854aee26f9e7f668372c47572d2b6d4f0f

doc/source/configuration.rst

@@ -280,6 +280,15 @@ with the following command:
 
     $ keystone-manage mapping_purge --all
 
+Generating public IDs in the first run may take a while, and most probably
+first API requests to fetch user list will fail by timeout. To prevent this,
+``mapping_populate`` command should be executed. It should be executed right after
+LDAP has been configured or after ``mapping_purge``.
+
+.. code-block:: bash
+
+    $ keystone-manage mapping_populate --domain DOMAINA
+
 Public ID Generators
 --------------------
 
@@ -1308,6 +1317,7 @@ through the normal REST API. At the moment, the following calls are supported:
 * ``fernet_rotate``: Rotate keys in the Fernet key repository.
 * ``fernet_setup``: Setup a Fernet key repository.
 * ``mapping_engine``: Test your federation mapping rules.
+* ``mapping_populate``: Prepare domain-specific LDAP backend
 * ``mapping_purge``: Purge the identity mapping table.
 * ``pki_setup``: Initialize the certificates used to sign tokens.
 * ``saml_idp_metadata``: Generate identity provider metadata.

doc/source/man/keystone-manage.rst

@@ -47,6 +47,7 @@ Available commands:
 * ``domain_config_upload``: Upload domain configuration file.
 * ``fernet_rotate``: Rotate keys in the Fernet key repository.
 * ``fernet_setup``: Setup a Fernet key repository.
+* ``mapping_populate``: Prepare domain-specific LDAP backend.
 * ``mapping_purge``: Purge the identity mapping table.
 * ``mapping_engine``: Test your federation mapping rules.
 * ``pki_setup``: Initialize the certificates used to sign tokens. **deprecated**

keystone/cmd/cli.py

@@ -1004,6 +1004,52 @@ class MappingEngineTester(BaseApp):
                                   "engine."))
 
 
+class MappingPopulate(BaseApp):
+    """Pre-populate entries from domain-specific backends.
+
+    Running this command is not required. It should only be run right after
+    the LDAP was configured, when many new users were added, or when
+    "mapping_purge" is run.
+
+    This command will take a while to run. It is perfectly fine for it to run
+    more than several minutes.
+    """
+
+    name = "mapping_populate"
+
+    @classmethod
+    def load_backends(cls):
+        drivers = backends.load_backends()
+        cls.identity_api = drivers['identity_api']
+        cls.resource_api = drivers['resource_api']
+
+    @classmethod
+    def add_argument_parser(cls, subparsers):
+        parser = super(MappingPopulate, cls).add_argument_parser(
+            subparsers)
+
+        parser.add_argument('--domain-name', default=None, required=True,
+                            help=("Name of the domain configured to use"
+                                  "domain-specific backend"))
+        return parser
+
+    @classmethod
+    def main(cls):
+        """Process entries for id_mapping_api."""
+        cls.load_backends()
+        domain_name = CONF.command.domain_name
+        try:
+            domain_id = cls.resource_api.get_domain_by_name(domain_name)['id']
+        except exception.DomainNotFound:
+            print(_('Invalid domain name or ID: %(domain)s') % {
+                'domain': domain_id})
+            return False
+        # We don't actually need to tackle id_mapping_api in order to get
+        # entries there, because list_users does this anyway. That's why it
+        # will be enough to just make the call below.
+        cls.identity_api.list_users(domain_scope=domain_id)
+
+
 CMDS = [
     BootStrap,
     DbSync,
@@ -1012,6 +1058,7 @@ CMDS = [
     DomainConfigUpload,
     FernetRotate,
     FernetSetup,
+    MappingPopulate,
     MappingPurge,
     MappingEngineTester,
     PKISetup,

keystone/tests/unit/test_cli.py

@@ -28,8 +28,11 @@ from keystone.common import dependency
 from keystone.common.sql import migration_helpers
 import keystone.conf
 from keystone.i18n import _
+from keystone.identity.mapping_backends import mapping as identity_mapping
 from keystone.tests import unit
+from keystone.tests.unit import default_fixtures
 from keystone.tests.unit.ksfixtures import database
+from keystone.tests.unit.ksfixtures import ldapdb
 
 
 CONF = keystone.conf.CONF
@@ -596,3 +599,64 @@ class CliDBSyncTestCase(unit.BaseTestCase):
             CONF, 'command', self.FakeConfCommand(self)))
         cli.DbSync.main()
         self._assert_correct_call(migration_helpers.contract_schema)
+
+
+class TestMappingPopulate(unit.SQLDriverOverrides, unit.TestCase):
+
+    def setUp(self):
+        sqldb = self.useFixture(database.Database())
+        super(TestMappingPopulate, self).setUp()
+        self.ldapdb = self.useFixture(ldapdb.LDAPDatabase())
+        self.ldapdb.clear()
+
+        self.load_backends()
+
+        sqldb.recreate()
+        self.load_fixtures(default_fixtures)
+
+    def config_files(self):
+        self.config_fixture.register_cli_opt(cli.command_opt)
+        config_files = super(TestMappingPopulate, self).config_files()
+        config_files.append(unit.dirs.tests_conf('backend_ldap_sql.conf'))
+        return config_files
+
+    def config_overrides(self):
+        super(TestMappingPopulate, self).config_overrides()
+        self.config_fixture.config(group='identity', driver='ldap')
+        self.config_fixture.config(group='identity_mapping',
+                                   backward_compatible_ids=False)
+
+    def config(self, config_files):
+        CONF(args=['mapping_populate', '--domain-name', 'Default'],
+             project='keystone',
+             default_config_files=config_files)
+
+    def test_mapping_populate(self):
+        # mapping_populate should create id mappings. Test plan:
+        # 0. Purge mappings
+        # 1. Fetch user list directly via backend. It will not create any
+        #    mappings because it bypasses identity manager
+        # 2. Verify that users have no public_id yet
+        # 3. Execute mapping_populate. It should create id mappings
+        # 4. For the same users verify that they have public_id now
+        purge_filter = {}
+        self.id_mapping_api.purge_mappings(purge_filter)
+        hints = None
+        users = self.identity_api.driver.list_users(hints)
+        for user in users:
+            local_entity = {
+                'domain_id': CONF.identity.default_domain_id,
+                'local_id': user['id'],
+                'entity_type': identity_mapping.EntityType.USER}
+            self.assertIsNone(self.id_mapping_api.get_public_id(local_entity))
+
+        dependency.reset()  # backends are loaded again in the command handler
+        cli.MappingPopulate.main()
+
+        for user in users:
+            local_entity = {
+                'domain_id': CONF.identity.default_domain_id,
+                'local_id': user['id'],
+                'entity_type': identity_mapping.EntityType.USER}
+            self.assertIsNotNone(
+                self.id_mapping_api.get_public_id(local_entity))

releasenotes/notes/mapping_populate-521d92445505b8a3.yaml

@@ -0,0 +1,13 @@
+---
+prelude: >
+    Add ``keystone-manage mapping_populate`` command, which
+    should be used when domain-specific LDAP backend is
+    used.
+features:
+  - Add ``keystone-manage mapping_populate`` command.
+    This command will pre-populate a mapping table with all
+    users from LDAP, in order to improve future query
+    performance. It should be used when an LDAP is first
+    configured, or after calling ``keystone-manage mapping_purge``,
+    before any queries related to the domain are made. For more
+    information see ``keystone-manage mapping_populate --help``