Merge "Account generator for identity v3"

tempest/cmd/account_generator.py

@@ -40,13 +40,15 @@ through CLI options or environment variables.
 
 You're probably familiar with these, but just to remind::
 
-    +----------+------------------+----------------------+
+    +----------+---------------------------+----------------------+
-    | Param    | CLI              | Environment Variable |
+    | Param    | CLI                       | Environment Variable |
-    +----------+------------------+----------------------+
+    +----------+---------------------------+----------------------+
-    | Username | --os-username    | OS_USERNAME          |
+    | Username | --os-username             | OS_USERNAME          |
-    | Password | --os-password    | OS_PASSWORD          |
+    | Password | --os-password             | OS_PASSWORD          |
-    | Tenant   | --os-tenant-name | OS_TENANT_NAME       |
+    | Project  | --os-project-name         | OS_PROJECT_NAME      |
-    +----------+------------------+----------------------+
+    | Tenant   | --os-tenant-name (depr.)  | OS_TENANT_NAME       |
+    | Domain   | --os-domain-name          | OS_DOMAIN_NAME       |
+    +----------+---------------------------+----------------------+
 
 Optional Arguments
 ------------------
@@ -63,8 +65,14 @@ have permissions to create new user accounts and tenants.
 **--os-password <auth-password>** (Optional) Password used for authentication
 with the OpenStack Identity service. Defaults to env[OS_PASSWORD].
 
-**--os-tenant-name <auth-tenant-name>** (Optional) Tenant to request
+**--os-project-name <auth-project-name>** (Optional) Project to request
-authorization on. Defaults to env[OS_TENANT_NAME].
+authorization on. Defaults to env[OS_PROJECT_NAME].
+
+**--os-tenant-name <auth-tenant-name>** (Optional, deprecated) Tenant to
+request authorization on. Defaults to env[OS_TENANT_NAME].
+
+**--os-domain-name <auth-domain-name>** (Optional) Domain the user and project
+belong to. Defaults to env[OS_DOMAIN_NAME].
 
 **--tag TAG** (Optional) Resources tag. Each created resource (user, project)
 will have the prefix with the given TAG in its name. Using tag is recommended
@@ -79,11 +87,13 @@ running in parallel.
 **--with-admin** (Optional) Creates admin for each concurrent group
 (default: False).
 
+**-i VERSION**, **--identity-version VERSION** (Optional) Provisions accounts
+using the specified version of the identity API. (default: '3').
+
 To see help on specific argument, please do: ``tempest-account-generator
 [OPTIONS] <accounts_file.yaml> -h``.
 """
 import argparse
-import netaddr
 import os
 import traceback
 
@@ -91,19 +101,10 @@ from cliff import command
 from oslo_log import log as logging
 import yaml
 
-from tempest.common import identity
+from tempest.common import credentials_factory
+from tempest.common import dynamic_creds
 from tempest import config
-from tempest import exceptions as exc
+
-import tempest.lib.auth
-from tempest.lib.common.utils import data_utils
-import tempest.lib.exceptions
-from tempest.lib.services.network import networks_client
-from tempest.lib.services.network import routers_client
-from tempest.lib.services.network import subnets_client
-from tempest.services.identity.v2.json import identity_client
-from tempest.services.identity.v2.json import roles_client
-from tempest.services.identity.v2.json import tenants_client
-from tempest.services.identity.v2.json import users_client
 
 LOG = None
 CONF = config.CONF
@@ -120,290 +121,83 @@ def setup_logging():
     LOG = logging.getLogger(__name__)
 
 
-def get_admin_clients(opts):
+def get_credential_provider(opts):
-    _creds = tempest.lib.auth.KeystoneV2Credentials(
+    identity_version = "".join(['v', str(opts.identity_version)])
-        username=opts.os_username,
+    # NOTE(andreaf) For now tempest.conf controls whether resources will
-        password=opts.os_password,
+    # actually be created. Once we remove the dependency from tempest.conf
-        tenant_name=opts.os_tenant_name)
+    # we will need extra CLI option(s) to control this.
-    auth_params = {
+    network_resources = {'router': True,
-        'disable_ssl_certificate_validation':
+                         'network': True,
-            CONF.identity.disable_ssl_certificate_validation,
+                         'subnet': True,
-        'ca_certs': CONF.identity.ca_certificates_file,
+                         'dhcp': True}
-        'trace_requests': CONF.debug.trace_requests
+    admin_creds_dict = {'username': opts.os_username,
-    }
+                        'password': opts.os_password}
-    _auth = tempest.lib.auth.KeystoneV2AuthProvider(
+    _project_name = opts.os_project_name or opts.os_tenant_name
-        _creds, CONF.identity.uri, **auth_params)
+    if opts.identity_version == 3:
-    params = {
+        admin_creds_dict['project_name'] = _project_name
-        'disable_ssl_certificate_validation':
+        admin_creds_dict['domain_name'] = opts.os_domain_name or 'Default'
-            CONF.identity.disable_ssl_certificate_validation,
+    elif opts.identity_version == 2:
-        'ca_certs': CONF.identity.ca_certificates_file,
+        admin_creds_dict['tenant_name'] = _project_name
-        'trace_requests': CONF.debug.trace_requests,
+    admin_creds = credentials_factory.get_credentials(
-        'build_interval': CONF.compute.build_interval,
+        fill_in=False, identity_version=identity_version, **admin_creds_dict)
-        'build_timeout': CONF.compute.build_timeout
+    return dynamic_creds.DynamicCredentialProvider(
-    }
+        identity_version=identity_version,
-    identity_admin = identity_client.IdentityClient(
+        name=opts.tag,
-        _auth,
+        network_resources=network_resources,
-        CONF.identity.catalog_type,
+        admin_creds=admin_creds,
-        CONF.identity.region,
+        **credentials_factory.get_dynamic_provider_params())
-        endpoint_type='adminURL',
-        **params
-    )
-    tenants_admin = tenants_client.TenantsClient(
-        _auth,
-        CONF.identity.catalog_type,
-        CONF.identity.region,
-        endpoint_type='adminURL',
-        **params
-    )
-    roles_admin = roles_client.RolesClient(
-        _auth,
-        CONF.identity.catalog_type,
-        CONF.identity.region,
-        endpoint_type='adminURL',
-        **params
-    )
-    users_admin = users_client.UsersClient(
-        _auth,
-        CONF.identity.catalog_type,
-        CONF.identity.region,
-        endpoint_type='adminURL',
-        **params
-    )
-    networks_admin = None
-    routers_admin = None
-    subnets_admin = None
-    neutron_iso_networks = False
-    if (CONF.service_available.neutron and
-        CONF.auth.create_isolated_networks):
-        neutron_iso_networks = True
-        networks_admin = networks_client.NetworksClient(
-            _auth,
-            CONF.network.catalog_type,
-            CONF.network.region or CONF.identity.region,
-            endpoint_type='adminURL',
-            **params)
-        routers_admin = routers_client.RoutersClient(
-            _auth,
-            CONF.network.catalog_type,
-            CONF.network.region or CONF.identity.region,
-            endpoint_type='adminURL',
-            **params)
-        subnets_admin = subnets_client.SubnetsClient(
-            _auth,
-            CONF.network.catalog_type,
-            CONF.network.region or CONF.identity.region,
-            endpoint_type='adminURL',
-            **params)
-    return (identity_admin, tenants_admin, roles_admin, users_admin,
-            neutron_iso_networks, networks_admin, routers_admin,
-            subnets_admin)
 
 
-def create_resources(opts, resources):
+def generate_resources(cred_provider, admin):
-    (identity_admin, tenants_admin, roles_admin, users_admin,
+    # Create the list of resources to be provisioned for each process
-     neutron_iso_networks, networks_admin, routers_admin,
+    # NOTE(andreaf) get_credentials expects a string for types or a list for
-     subnets_admin) = get_admin_clients(opts)
+    # roles. Adding all required inputs to the spec list.
-    roles = roles_admin.list_roles()['roles']
+    spec = ['primary', 'alt']
-    for u in resources['users']:
-        u['role_ids'] = []
-        for r in u.get('roles', ()):
-            try:
-                role = filter(lambda r_: r_['name'] == r, roles)[0]
-            except IndexError:
-                msg = "Role: %s doesn't exist" % r
-                raise exc.InvalidConfiguration(msg)
-            u['role_ids'] += [role['id']]
-    existing = [x['name'] for x in tenants_admin.list_tenants()['tenants']]
-    for tenant in resources['tenants']:
-        if tenant not in existing:
-            tenants_admin.create_tenant(tenant)
-        else:
-            LOG.warning("Tenant '%s' already exists in this environment"
-                        % tenant)
-    LOG.info('Tenants created')
-    for u in resources['users']:
-        try:
-            tenant = identity.get_tenant_by_name(tenants_admin, u['tenant'])
-        except tempest.lib.exceptions.NotFound:
-            LOG.error("Tenant: %s - not found" % u['tenant'])
-            continue
-        while True:
-            try:
-                identity.get_user_by_username(tenants_admin,
-                                              tenant['id'], u['name'])
-            except tempest.lib.exceptions.NotFound:
-                users_admin.create_user(
-                    u['name'], u['pass'], tenant['id'],
-                    "%s@%s" % (u['name'], tenant['id']),
-                    enabled=True)
-                break
-            else:
-                LOG.warning("User '%s' already exists in this environment. "
-                            "New name generated" % u['name'])
-                u['name'] = random_user_name(opts.tag, u['prefix'])
-
-    LOG.info('Users created')
-    if neutron_iso_networks:
-        for u in resources['users']:
-            tenant = identity.get_tenant_by_name(tenants_admin, u['tenant'])
-            network_name, router_name = create_network_resources(
-                networks_admin, routers_admin, subnets_admin,
-                tenant['id'], u['name'])
-            u['network'] = network_name
-            u['router'] = router_name
-        LOG.info('Networks created')
-    for u in resources['users']:
-        try:
-            tenant = identity.get_tenant_by_name(tenants_admin, u['tenant'])
-        except tempest.lib.exceptions.NotFound:
-            LOG.error("Tenant: %s - not found" % u['tenant'])
-            continue
-        try:
-            user = identity.get_user_by_username(tenants_admin,
-                                                 tenant['id'], u['name'])
-        except tempest.lib.exceptions.NotFound:
-            LOG.error("User: %s - not found" % u['name'])
-            continue
-        for r in u['role_ids']:
-            try:
-                roles_admin.assign_user_role(tenant['id'], user['id'], r)
-            except tempest.lib.exceptions.Conflict:
-                # don't care if it's already assigned
-                pass
-    LOG.info('Roles assigned')
-    LOG.info('Resources deployed successfully!')
-
-
-def create_network_resources(networks_admin_client,
-                             routers_admin_client, subnets_admin_client,
-                             tenant_id, name):
-
-    def _create_network(name):
-        resp_body = networks_admin_client.create_network(
-            name=name, tenant_id=tenant_id)
-        return resp_body['network']
-
-    def _create_subnet(subnet_name, network_id):
-        base_cidr = netaddr.IPNetwork(CONF.network.project_network_cidr)
-        mask_bits = CONF.network.project_network_mask_bits
-        for subnet_cidr in base_cidr.subnet(mask_bits):
-            try:
-                resp_body = subnets_admin_client.\
-                    create_subnet(
-                        network_id=network_id, cidr=str(subnet_cidr),
-                        name=subnet_name,
-                        tenant_id=tenant_id,
-                        enable_dhcp=True,
-                        ip_version=4)
-                break
-            except tempest.lib.exceptions.BadRequest as e:
-                if 'overlaps with another subnet' not in str(e):
-                    raise
-        else:
-            message = 'Available CIDR for subnet creation could not be found'
-            raise Exception(message)
-        return resp_body['subnet']
-
-    def _create_router(router_name):
-        external_net_id = dict(
-            network_id=CONF.network.public_network_id)
-        resp_body = routers_admin_client.create_router(
-            name=router_name,
-            external_gateway_info=external_net_id,
-            tenant_id=tenant_id)
-        return resp_body['router']
-
-    def _add_router_interface(router_id, subnet_id):
-        routers_admin_client.add_router_interface(router_id,
-                                                  subnet_id=subnet_id)
-
-    network_name = name + "-network"
-    network = _create_network(network_name)
-    subnet_name = name + "-subnet"
-    subnet = _create_subnet(subnet_name, network['id'])
-    router_name = name + "-router"
-    router = _create_router(router_name)
-    _add_router_interface(router['id'], subnet['id'])
-    return network_name, router_name
-
-
-def random_user_name(tag, prefix):
-    if tag:
-        return data_utils.rand_name('-'.join((tag, prefix)))
-    else:
-        return data_utils.rand_name(prefix)
-
-
-def generate_resources(opts):
-    spec = [{'number': 1,
-             'prefix': 'primary',
-             'roles': (CONF.auth.tempest_roles +
-                       [CONF.object_storage.operator_role])},
-            {'number': 1,
-             'prefix': 'alt',
-             'roles': (CONF.auth.tempest_roles +
-                       [CONF.object_storage.operator_role])}]
     if CONF.service_available.swift:
-        spec.append({'number': 1,
+        spec.append([CONF.object_storage.operator_role])
-                     'prefix': 'swift_operator',
+        spec.append([CONF.object_storage.reseller_admin_role])
-                     'roles': (CONF.auth.tempest_roles +
-                               [CONF.object_storage.operator_role])})
-        spec.append({'number': 1,
-                     'prefix': 'swift_reseller_admin',
-                     'roles': (CONF.auth.tempest_roles +
-                               [CONF.object_storage.reseller_admin_role])})
     if CONF.service_available.heat:
-        spec.append({'number': 1,
+        spec.append([CONF.orchestration.stack_owner_role])
-                     'prefix': 'stack_owner',
+    if admin:
-                     'roles': (CONF.auth.tempest_roles +
+        spec.append('admin')
-                               [CONF.orchestration.stack_owner_role])})
+    resources = []
-    if opts.admin:
+    for cred_type in spec:
-        spec.append({
+        resources.append((cred_type, cred_provider.get_credentials(
-            'number': 1,
+            credential_type=cred_type)))
-            'prefix': 'admin',
-            'roles': (CONF.auth.tempest_roles +
-                      [CONF.identity.admin_role])
-        })
-    resources = {'tenants': [],
-                 'users': []}
-    for count in range(opts.concurrency):
-        for user_group in spec:
-            users = [random_user_name(opts.tag, user_group['prefix'])
-                     for _ in range(user_group['number'])]
-            for user in users:
-                tenant = '-'.join((user, 'tenant'))
-                resources['tenants'].append(tenant)
-                resources['users'].append({
-                    'tenant': tenant,
-                    'name': user,
-                    'pass': data_utils.rand_password(),
-                    'prefix': user_group['prefix'],
-                    'roles': user_group['roles']
-                })
     return resources
 
 
-def dump_accounts(opts, resources):
+def dump_accounts(resources, identity_version, account_file):
     accounts = []
-    for user in resources['users']:
+    for resource in resources:
+        cred_type, test_resource = resource
         account = {
-            'username': user['name'],
+            'username': test_resource.username,
-            'tenant_name': user['tenant'],
+            'password': test_resource.password
-            'password': user['pass'],
-            'roles': user['roles']
         }
-        if 'network' in user or 'router' in user:
+        if identity_version == 3:
+            account['project_name'] = test_resource.project_name
+            account['domain_name'] = test_resource.domain_name
+        else:
+            account['project_name'] = test_resource.tenant_name
+
+        # If the spec includes 'admin' credentials are defined via type,
+        # else they are defined via list of roles.
+        if cred_type == 'admin':
+            account['types'] = [cred_type]
+        elif cred_type not in ['primary', 'alt']:
+            account['roles'] = cred_type
+
+        if test_resource.network:
             account['resources'] = {}
-        if 'network' in user:
+        if test_resource.network:
-            account['resources']['network'] = user['network']
+            account['resources']['network'] = test_resource.network['name']
-        if 'router' in user:
-            account['resources']['router'] = user['router']
         accounts.append(account)
-    if os.path.exists(opts.accounts):
+    if os.path.exists(account_file):
-        os.rename(opts.accounts, '.'.join((opts.accounts, 'bak')))
+        os.rename(account_file, '.'.join((account_file, 'bak')))
-    with open(opts.accounts, 'w') as f:
+    with open(account_file, 'w') as f:
-        yaml.dump(accounts, f, default_flow_style=False)
+        yaml.safe_dump(accounts, f, default_flow_style=False)
-    LOG.info('%s generated successfully!' % opts.accounts)
+    LOG.info('%s generated successfully!' % account_file)
 
 
 def _parser_add_args(parser):
@@ -420,10 +214,18 @@ def _parser_add_args(parser):
                         metavar='<auth-password>',
                         default=os.environ.get('OS_PASSWORD'),
                         help='Defaults to env[OS_PASSWORD].')
+    parser.add_argument('--os-project-name',
+                        metavar='<auth-project-name>',
+                        default=os.environ.get('OS_PROJECT_NAME'),
+                        help='Defaults to env[OS_PROJECT_NAME].')
     parser.add_argument('--os-tenant-name',
                         metavar='<auth-tenant-name>',
                         default=os.environ.get('OS_TENANT_NAME'),
                         help='Defaults to env[OS_TENANT_NAME].')
+    parser.add_argument('--os-domain-name',
+                        metavar='<auth-domain-name>',
+                        default=os.environ.get('OS_DOMAIN_NAME'),
+                        help='Defaults to env[OS_DOMAIN_NAME].')
     parser.add_argument('--tag',
                         default='',
                         required=False,
@@ -439,6 +241,13 @@ def _parser_add_args(parser):
                         action='store_true',
                         dest='admin',
                         help='Creates admin for each concurrent group')
+    parser.add_argument('-i', '--identity-version',
+                        default=3,
+                        choices=[2, 3],
+                        type=int,
+                        required=False,
+                        dest='identity_version',
+                        help='Version of the Identity API to use')
     parser.add_argument('accounts',
                         metavar='accounts_file.yaml',
                         help='Output accounts yaml file')
@@ -487,9 +296,16 @@ def main(opts=None):
         opts = get_options()
     if opts.config_file:
         config.CONF.set_config_path(opts.config_file)
-    resources = generate_resources(opts)
+    if opts.os_tenant_name:
-    create_resources(opts, resources)
+        LOG.warning("'os-tenant-name' and 'OS_TENANT_NAME' are both "
-    dump_accounts(opts, resources)
+                    "deprecated, please use 'os-project-name' or "
+                    "'OS_PROJECT_NAME' instead")
+    resources = []
+    for count in range(opts.concurrency):
+        # Use N different cred_providers to obtain different sets of creds
+        cred_provider = get_credential_provider(opts)
+        resources.extend(generate_resources(cred_provider, opts.admin))
+    dump_accounts(resources, opts.identity_version, opts.accounts)
 
 if __name__ == "__main__":
     main()

tempest/common/cred_client.py

@@ -155,6 +155,9 @@ class V3CredsClient(CredsClient):
     def get_credentials(self, user, project, password):
         # User, project and domain already include both ID and name here,
         # so there's no need to use the fill_in mode.
+        # NOTE(andreaf) We need to set all fields in the returned credentials.
+        # Scope is then used to pick only those relevant for the type of
+        # token needed by each service client.
         return auth.get_credentials(
             auth_url=None,
             fill_in=False,
@@ -163,7 +166,9 @@ class V3CredsClient(CredsClient):
             project_name=project['name'], project_id=project['id'],
             password=password,
             project_domain_id=self.creds_domain['id'],
-            project_domain_name=self.creds_domain['name'])
+            project_domain_name=self.creds_domain['name'],
+            domain_id=self.creds_domain['id'],
+            domain_name=self.creds_domain['name'])
 
     def _assign_user_role(self, project, user, role):
         self.roles_client.assign_user_role_on_project(project['id'],

tempest/common/credentials_factory.py

@@ -39,19 +39,19 @@ to avoid circular dependencies."""
 
 
 # Subset of the parameters of credential providers that depend on configuration
-def _get_common_provider_params():
+def get_common_provider_params():
     return {
         'credentials_domain': CONF.auth.default_credentials_domain_name,
         'admin_role': CONF.identity.admin_role
     }
 
 
-def _get_dynamic_provider_params():
+def get_dynamic_provider_params():
-    return _get_common_provider_params()
+    return get_common_provider_params()
 
 
-def _get_preprov_provider_params():
+def get_preprov_provider_params():
-    _common_params = _get_common_provider_params()
+    _common_params = get_common_provider_params()
     reseller_admin_role = CONF.object_storage.reseller_admin_role
     return dict(_common_params, **dict([
         ('accounts_lock_dir', lockutils.get_lock_path(CONF)),
@@ -80,13 +80,13 @@ def get_credentials_provider(name, network_resources=None,
             network_resources=network_resources,
             identity_version=identity_version,
             admin_creds=admin_creds,
-            **_get_dynamic_provider_params())
+            **get_dynamic_provider_params())
     else:
         if CONF.auth.test_accounts_file:
             # Most params are not relevant for pre-created accounts
             return preprov_creds.PreProvisionedCredentialProvider(
                 name=name, identity_version=identity_version,
-                **_get_preprov_provider_params())
+                **get_preprov_provider_params())
         else:
             raise exceptions.InvalidConfiguration(
                 'A valid credential provider is needed')
@@ -106,7 +106,7 @@ def is_admin_available(identity_version):
     elif CONF.auth.test_accounts_file:
         check_accounts = preprov_creds.PreProvisionedCredentialProvider(
             identity_version=identity_version, name='check_admin',
-            **_get_preprov_provider_params())
+            **get_preprov_provider_params())
         if not check_accounts.admin_available():
             is_admin = False
     else:
@@ -131,7 +131,7 @@ def is_alt_available(identity_version):
     if CONF.auth.test_accounts_file:
         check_accounts = preprov_creds.PreProvisionedCredentialProvider(
             identity_version=identity_version, name='check_alt',
-            **_get_preprov_provider_params())
+            **get_preprov_provider_params())
     else:
         raise exceptions.InvalidConfiguration(
             'A valid credential provider is needed')