[Fix gate] Fix failing identity v2 admin tests

This patch fixes the failing identity tests that
belong to the v2 admin API [0].

Due to recent changes to Keystone and DevStack,
the v2 admin API tests fail if run outright.

Tempest now skips the v2 admin tests uness
``CONF.identity_feature_enabled.api_v2_admin`` is true [1];
Patrole should do the same.

This patch makes the following changes:
  - Skips identity v2 admin tests unless
    ``CONF.identity_feature_enabled.api_v2_admin`` is true
  - Removes superfluous tempest.conf overrides from
    post_test_hook
  - Updates ``rbac_utils.switch_role`` to ensure that admin
    identity credentials are properly configured and that
    the ``roles_v3_client`` is always used
  - Refactors test_projects_rbac in v2 identity because
    the API belongs to both the admin and non-admin API,
    which was causing OverPermission error to be thrown.

[0] https://developer.openstack.org/api-ref/identity/v2-admin/
[1] https://review.openstack.org/#/c/458844/

Change-Id: Ic698d0b2cf669793aaad6aff972ba155ef993e4a

contrib/post_test_hook.sh

@@ -61,9 +61,7 @@ function setup_config() {
     # Set strict_policy_check=False under [rbac] section in tempest.conf
     iniset $TEMPEST_CONFIG rbac strict_policy_check False
     # Set additional, necessary CONF values
-    iniset $TEMPEST_CONFIG auth use_dynamic_credentials True
     iniset $TEMPEST_CONFIG auth tempest_roles Member
-    iniset $TEMPEST_CONFIG identity auth_version v3
 }
 
 function run_tests() {

patrole_tempest_plugin/rbac_rule_validation.py

@@ -29,7 +29,7 @@ CONF = config.CONF
 LOG = logging.getLogger(__name__)
 
 
-def action(service, rule, admin_only=False, expected_error_code=403,
+def action(service, rule='', admin_only=False, expected_error_code=403,
            extra_target_data={}):
     """A decorator which does a policy check and matches it against test run.
 

patrole_tempest_plugin/rbac_utils.py

@@ -20,6 +20,7 @@ from oslo_log import log as logging
 import oslo_utils.uuidutils as uuid_utils
 import six
 
+from tempest.common import credentials_factory as credentials
 from tempest import config
 
 from patrole_tempest_plugin import rbac_exceptions
@@ -56,10 +57,12 @@ class RbacUtils(object):
         self.token = test_obj.auth_provider.get_token()
         self.identity_version = test_obj.get_identity_version()
 
-        if self.identity_version.endswith('v3'):
+        if not credentials.is_admin_available(
-            self.roles_client = test_obj.os_admin.roles_v3_client
+                identity_version=self.identity_version):
-        else:
+            msg = "Missing Identity Admin API credentials in configuration."
-            self.roles_client = test_obj.os_admin.roles_client
+            raise rbac_exceptions.RbacResourceSetupFailed(msg)
+
+        self.roles_client = test_obj.os_admin.roles_v3_client
 
         LOG.debug('Switching role to: %s', toggle_rbac_role)
 

patrole_tempest_plugin/tests/api/identity/v2/rbac_base.py

@@ -23,20 +23,30 @@ from patrole_tempest_plugin.rbac_utils import rbac_utils
 CONF = config.CONF
 
 
-class BaseIdentityV2RbacTest(base.BaseIdentityV2Test):
+class BaseIdentityV2AdminRbacTest(base.BaseIdentityV2Test):
+    """Base test class for the Identity v2 admin API.
+
+    Keystone's v2 API is split into two APIs: an admin and non-admin API. RBAC
+    testing is only provided for the admin API. Instead of policy enforcement,
+    these APIs execute ``self.assert_admin(request)``, which checks that the
+    request object has ``context_is_admin``. For more details, see the
+    implementation of ``assert_admin`` in ``keystone.common.wsgi``.
+    """
 
     credentials = ['admin', 'primary']
 
     @classmethod
     def skip_checks(cls):
-        super(BaseIdentityV2RbacTest, cls).skip_checks()
+        super(BaseIdentityV2AdminRbacTest, cls).skip_checks()
         if not CONF.rbac.enable_rbac:
             raise cls.skipException(
                 "%s skipped as RBAC testing not enabled" % cls.__name__)
+        if not CONF.identity_feature_enabled.api_v2_admin:
+            raise cls.skipException('Identity v2 admin not available')
 
     @classmethod
     def setup_clients(cls):
-        super(BaseIdentityV2RbacTest, cls).setup_clients()
+        super(BaseIdentityV2AdminRbacTest, cls).setup_clients()
         cls.auth_provider = cls.os_primary.auth_provider
 
         cls.rbac_utils = rbac_utils()

patrole_tempest_plugin/tests/api/identity/v2/test_endpoints_rbac.py

@@ -13,7 +13,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from tempest import config
 from tempest.lib.common.utils import data_utils
 from tempest.lib.common.utils import test_utils
 from tempest.lib import decorators
@@ -21,19 +20,17 @@ from tempest.lib import decorators
 from patrole_tempest_plugin import rbac_rule_validation
 from patrole_tempest_plugin.tests.api.identity.v2 import rbac_base
 
-CONF = config.CONF
 
-
+class IdentityEndpointsV2AdminRbacTest(rbac_base.BaseIdentityV2AdminRbacTest):
-class IdentityEndpointsV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
 
     @classmethod
     def setup_clients(cls):
-        super(IdentityEndpointsV2RbacTest, cls).setup_clients()
+        super(IdentityEndpointsV2AdminRbacTest, cls).setup_clients()
         cls.endpoints_client = cls.os_primary.endpoints_client
 
     @classmethod
     def resource_setup(cls):
-        super(IdentityEndpointsV2RbacTest, cls).resource_setup()
+        super(IdentityEndpointsV2AdminRbacTest, cls).resource_setup()
         cls.region = data_utils.rand_name('region')
         cls.public_url = data_utils.rand_url()
         cls.admin_url = data_utils.rand_url()
@@ -54,7 +51,6 @@ class IdentityEndpointsV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
         return endpoint
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:create_endpoint",
                                  admin_only=True)
     @decorators.idempotent_id('6bdaecd4-0843-4ed6-ab64-3a57ab0cd124')
     def test_create_endpoint(self):
@@ -68,7 +64,6 @@ class IdentityEndpointsV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
         self._create_endpoint()
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:delete_endpoint",
                                  admin_only=True)
     @decorators.idempotent_id('6bdaecd4-0843-4ed6-ab64-3a57ab0cd125')
     def test_delete_endpoint(self):
@@ -83,7 +78,6 @@ class IdentityEndpointsV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
         self.endpoints_client.delete_endpoint(endpoint['endpoint']['id'])
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:list_endpoints",
                                  admin_only=True)
     @decorators.idempotent_id('6bdaecd4-0843-4ed6-ab64-3a57ab0cd126')
     def test_list_endpoints(self):

patrole_tempest_plugin/tests/api/identity/v2/test_projects_rbac.py

@@ -16,16 +16,16 @@
 from tempest import config
 from tempest.lib import decorators
 
+from patrole_tempest_plugin import rbac_exceptions
 from patrole_tempest_plugin import rbac_rule_validation
 from patrole_tempest_plugin.tests.api.identity.v2 import rbac_base
 
 CONF = config.CONF
 
 
-class IdentityProjectV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
+class IdentityProjectV2AdminRbacTest(rbac_base.BaseIdentityV2AdminRbacTest):
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:create_project",
                                  admin_only=True)
     @decorators.idempotent_id('0f148510-63bf-11e6-b348-080044d0d904')
     def test_create_project(self):
@@ -39,7 +39,6 @@ class IdentityProjectV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
         self._create_tenant()
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:update_project",
                                  admin_only=True)
     @decorators.idempotent_id('0f148510-63bf-11e6-b348-080044d0d905')
     def test_update_project(self):
@@ -55,7 +54,6 @@ class IdentityProjectV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
                                           description="Changed description")
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:delete_project",
                                  admin_only=True)
     @decorators.idempotent_id('0f148510-63bf-11e6-b348-080044d0d906')
     def test_delete_project(self):
@@ -70,7 +68,6 @@ class IdentityProjectV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
         self.tenants_client.delete_tenant(tenant['id'])
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:get_project",
                                  admin_only=True)
     @decorators.idempotent_id('0f148510-63bf-11e6-b348-080044d0d907')
     def test_get_project(self):
@@ -86,20 +83,6 @@ class IdentityProjectV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
         self.tenants_client.show_tenant(tenant['id'])
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:list_projects",
-                                 admin_only=True)
-    @decorators.idempotent_id('0f148510-63bf-11e6-b348-080044d0d908')
-    def test_get_all_projects(self):
-
-        """List All Projects Test
-
-        RBAC test for Identity 2.0 list_tenants
-        """
-        self.rbac_utils.switch_role(self, toggle_rbac_role=True)
-        self.tenants_client.list_tenants()
-
-    @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:list_user_projects",
                                  admin_only=True)
     @decorators.idempotent_id('0f148510-63bf-11e6-b348-080044d0d909')
     def test_list_project_users(self):
@@ -112,3 +95,37 @@ class IdentityProjectV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
 
         self.rbac_utils.switch_role(self, toggle_rbac_role=True)
         self.tenants_client.list_tenant_users(tenant['id'])
+
+    @rbac_rule_validation.action(service="keystone",
+                                 admin_only=True)
+    @decorators.idempotent_id('0f148510-63bf-11e6-b348-080044d0d908')
+    def test_list_all_projects(self):
+
+        """List All Projects Test
+
+        RBAC test for Identity 2.0 list_tenants (admin endpoint)
+
+        There are two separate APIs for listing tenants in the Keystone
+        v2 API: one for admin and one for non-admin. The ``os_admin`` client
+        calls the admin endpoint and the ``os_primary`` client calls the
+        non-admin endpoint. To ensure that the admin endpoint only returns
+        admin-scoped tenants, raise ``RbacActionFailed`` exception otherwise.
+        """
+        tenants_client = self.os_admin.tenants_client if \
+            CONF.identity.admin_role == CONF.rbac.rbac_test_role else \
+            self.os_primary.tenants_client
+        admin_tenant_id = self.os_admin.auth_provider.credentials.project_id
+        non_admin_tenant_id = self.auth_provider.credentials.project_id
+
+        self.rbac_utils.switch_role(self, toggle_rbac_role=True)
+        tenants = tenants_client.list_tenants()['tenants']
+
+        tenant_ids = [t['id'] for t in tenants]
+        if admin_tenant_id not in tenant_ids:
+            raise rbac_exceptions.RbacActionFailed(
+                "The admin tenant id was not returned by the call to "
+                "``list_tenants``.")
+        if non_admin_tenant_id in tenant_ids:
+            raise rbac_exceptions.RbacActionFailed(
+                "The non-admin tenant id was returned by the call to "
+                "``list_tenants``.")

patrole_tempest_plugin/tests/api/identity/v2/test_roles_rbac.py

@@ -13,7 +13,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from tempest import config
 from tempest.lib.common.utils import data_utils
 from tempest.lib.common.utils import test_utils
 from tempest.lib import decorators
@@ -21,14 +20,12 @@ from tempest.lib import decorators
 from patrole_tempest_plugin import rbac_rule_validation
 from patrole_tempest_plugin.tests.api.identity.v2 import rbac_base
 
-CONF = config.CONF
 
-
+class IdentityRolesV2AdminRbacTest(rbac_base.BaseIdentityV2AdminRbacTest):
-class IdentityRoleV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
 
     @classmethod
     def setup_clients(cls):
-        super(IdentityRoleV2RbacTest, cls).setup_clients()
+        super(IdentityRolesV2AdminRbacTest, cls).setup_clients()
         cls.roles_client = cls.os_primary.roles_client
 
     def _create_role(self):
@@ -53,7 +50,6 @@ class IdentityRoleV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
             tenant['id'], user['id'], role['id'])
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:create_role",
                                  admin_only=True)
     @decorators.idempotent_id('0f148510-63bf-11e6-8674-080044d0d904')
     def test_create_role(self):
@@ -67,7 +63,6 @@ class IdentityRoleV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
         self._create_role()
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:delete_role",
                                  admin_only=True)
     @decorators.idempotent_id('0f148510-63bf-11e6-8674-080044d0d905')
     def test_delete_role(self):
@@ -82,7 +77,6 @@ class IdentityRoleV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
         self.roles_client.delete_role(role['id'])
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:get_role",
                                  admin_only=True)
     @decorators.idempotent_id('0f148510-63bf-11e6-8674-080044d0d906')
     def test_show_role(self):
@@ -97,7 +91,6 @@ class IdentityRoleV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
         self.roles_client.show_role(role['id'])
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:list_roles",
                                  admin_only=True)
     @decorators.idempotent_id('0f148510-63bf-11e6-8674-080044d0d907')
     def test_list_roles(self):
@@ -110,7 +103,6 @@ class IdentityRoleV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
         self.roles_client.list_roles()
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:add_role_to_user",
                                  admin_only=True)
     @decorators.idempotent_id('0f148510-63bf-11e6-8674-080044d0d908')
     def test_create_role_on_project(self):
@@ -124,7 +116,6 @@ class IdentityRoleV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
         self._create_role_on_project(tenant, user, role)
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:remove_role_from_user",
                                  admin_only=True)
     @decorators.idempotent_id('0f148510-63bf-11e6-8674-080044d0d909')
     def test_delete_role_from_user_on_project(self):
@@ -141,7 +132,6 @@ class IdentityRoleV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
             tenant['id'], user['id'], role['id'])
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:get_user_roles",
                                  admin_only=True)
     @decorators.idempotent_id('0f148510-63bf-11e6-8674-080044d0d90a')
     def test_list_user_roles_on_project(self):

patrole_tempest_plugin/tests/api/identity/v2/test_services_rbac.py

@@ -13,24 +13,20 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from tempest import config
 from tempest.lib import decorators
 
 from patrole_tempest_plugin import rbac_rule_validation
 from patrole_tempest_plugin.tests.api.identity.v2 import rbac_base
 
-CONF = config.CONF
 
-
+class IdentityServicesV2AdminRbacTest(rbac_base.BaseIdentityV2AdminRbacTest):
-class IdentityServicesV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
 
     @classmethod
     def setup_clients(cls):
-        super(IdentityServicesV2RbacTest, cls).setup_clients()
+        super(IdentityServicesV2AdminRbacTest, cls).setup_clients()
         cls.services_client = cls.os_primary.identity_services_client
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:create_service",
                                  admin_only=True)
     @decorators.idempotent_id('370050f6-d271-4fb4-abc5-4de1d6dfbad2')
     def test_create_service(self):
@@ -42,7 +38,6 @@ class IdentityServicesV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
         self._create_service()
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:delete_service",
                                  admin_only=True)
     @decorators.idempotent_id('f6c64fc3-6a1f-423e-af91-e411add3a384')
     def test_delete_service(self):
@@ -56,7 +51,6 @@ class IdentityServicesV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
         self.services_client.delete_service(service_id)
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:get_service",
                                  admin_only=True)
     @decorators.idempotent_id('504d62bb-97d7-445e-9d6d-b1945a7c9e08')
     def test_show_service(self):
@@ -70,7 +64,6 @@ class IdentityServicesV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
         self.services_client.show_service(service_id)
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:list_services",
                                  admin_only=True)
     @decorators.idempotent_id('d7dc461d-51ad-48e0-9cd2-33add1b88de9')
     def test_list_services(self):

patrole_tempest_plugin/tests/api/identity/v2/test_users_rbac.py

@@ -20,10 +20,9 @@ from patrole_tempest_plugin import rbac_rule_validation
 from patrole_tempest_plugin.tests.api.identity.v2 import rbac_base
 
 
-class IdentityUserV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
+class IdentityUsersV2AdminRbacTest(rbac_base.BaseIdentityV2AdminRbacTest):
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:create_user",
                                  admin_only=True)
     @decorators.idempotent_id('0f148510-63bf-11e6-1342-080044d0d904')
     def test_create_user(self):
@@ -31,7 +30,6 @@ class IdentityUserV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
         self._create_user()
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:update_user",
                                  admin_only=True)
     @decorators.idempotent_id('0f148510-63bf-11e6-1342-080044d0d905')
     def test_update_user(self):
@@ -41,7 +39,6 @@ class IdentityUserV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
         self.users_client.update_user(user['id'], email="changedUser@xyz.com")
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:set_user_enabled",
                                  admin_only=True)
     @decorators.idempotent_id('0f148510-63bf-11e6-1342-080044d0d9a1')
     def test_update_user_enabled(self):
@@ -51,7 +48,6 @@ class IdentityUserV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
         self.users_client.update_user_enabled(user['id'], enabled=True)
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:delete_user",
                                  admin_only=True)
     @decorators.idempotent_id('0f148510-63bf-11e6-1342-080044d0d906')
     def test_delete_user(self):
@@ -61,7 +57,6 @@ class IdentityUserV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
         self.users_client.delete_user(user['id'])
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:get_users",
                                  admin_only=True)
     @decorators.idempotent_id('0f148510-63bf-11e6-1342-080044d0d907')
     def test_list_users(self):
@@ -69,7 +64,6 @@ class IdentityUserV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
         self.users_client.list_users()
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:get_user",
                                  admin_only=True)
     @decorators.idempotent_id('0f148510-63bf-11e6-1342-080044d0d908')
     def test_show_user(self):
@@ -79,7 +73,6 @@ class IdentityUserV2RbacTest(rbac_base.BaseIdentityV2RbacTest):
         self.users_client.show_user(user['id'])
 
     @rbac_rule_validation.action(service="keystone",
-                                 rule="identity:change_password",
                                  admin_only=True)
     @decorators.idempotent_id('0f148510-63bf-11e6-1342-080044d0d909')
     def test_update_user_password(self):

patrole_tempest_plugin/tests/api/identity/v3/test_users_rbac.py

@@ -20,11 +20,11 @@ from patrole_tempest_plugin import rbac_rule_validation
 from patrole_tempest_plugin.tests.api.identity.v3 import rbac_base
 
 
-class IdentityUserV3RbacTest(rbac_base.BaseIdentityV3RbacTest):
+class IdentityUserV3AdminRbacTest(rbac_base.BaseIdentityV3RbacTest):
 
     @classmethod
     def resource_setup(cls):
-        super(IdentityUserV3RbacTest, cls).resource_setup()
+        super(IdentityUserV3AdminRbacTest, cls).resource_setup()
         cls.default_user_id = cls.auth_provider.credentials.user_id
 
     @rbac_rule_validation.action(service="keystone",

releasenotes/notes/admin-only-identity-v2-admin-6f382e38d7a690a4.yaml

@@ -0,0 +1,12 @@
+---
+fixes:
+  - |
+    Removed ``rule`` kwarg from ``rbac_rule_validation`` decorator for identity
+    v2 admin tests, because the identity v2 admin API does not do policy
+    enforcement, and instead checks whether the request object has
+    ``context_is_admin``.
+other:
+  - |
+    Updated the class names for identity v2 tests to include the "Admin"
+    substring, to convey the fact that these tests are only intended
+    to test the v2 admin API, not the v2 API.