Fix gates against Keystone V3

Fix 'rally verify' against Keystone v3
Support keystone V3 in tests.ci.osresources
Disable Keystone and Ceilometer related scenarios

Change-Id: If352298e9ec6ddc0cc0a9e14f14b7218ed6d1707
Co-Authored-By: Alexander Maretskiy <amaretskiy@mirantis.com>

rally-jobs/rally.yaml

@@ -1,391 +1,391 @@
 {%- set cirros_image_url = "http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-disk.img" %}
 ---
-  KeystoneBasic.create_user:
+  #KeystoneBasic.create_user:
-    -
+    #-
-      args: {}
+      #args: {}
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 10
+        #concurrency: 10
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  KeystoneBasic.create_delete_user:
+  #KeystoneBasic.create_delete_user:
-    -
+    #-
-      args: {}
+      #args: {}
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 10
+        #concurrency: 10
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  KeystoneBasic.create_user_set_enabled_and_delete:
+  #KeystoneBasic.create_user_set_enabled_and_delete:
-    -
+    #-
-      args:
+      #args:
-        enabled: true
+        #enabled: true
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 10
+        #concurrency: 10
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
-    -
+    #-
-      args:
+      #args:
-        enabled: false
+        #enabled: false
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 10
+        #concurrency: 10
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  KeystoneBasic.create_and_list_tenants:
+  #KeystoneBasic.create_and_list_tenants:
-    -
+    #-
-      args: {}
+      #args: {}
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 10
+        #concurrency: 10
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  KeystoneBasic.get_entities:
+  #KeystoneBasic.get_entities:
-    -
+    #-
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 20
+        #times: 20
-        concurrency: 10
+        #concurrency: 10
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
-    -
+    #-
-      args:
+      #args:
-        service_name: null
+        #service_name: null
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 20
+        #times: 20
-        concurrency: 10
+        #concurrency: 10
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
-    -
+    #-
-      args:
+      #args:
-        service_name: "nova"
+        #service_name: "nova"
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 20
+        #times: 20
-        concurrency: 10
+        #concurrency: 10
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  KeystoneBasic.add_and_remove_user_role:
+  #KeystoneBasic.add_and_remove_user_role:
-    -
+    #-
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 5
+        #concurrency: 5
-      context:
+      #context:
-        users:
+        #users:
-          tenants: 1
+          #tenants: 1
-          users_per_tenant: 1
+          #users_per_tenant: 1
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  KeystoneBasic.create_and_delete_role:
+  #KeystoneBasic.create_and_delete_role:
-    -
+    #-
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 5
+        #concurrency: 5
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  KeystoneBasic.create_add_and_list_user_roles:
+  #KeystoneBasic.create_add_and_list_user_roles:
-    -
+    #-
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 5
+        #concurrency: 5
-      context:
+      #context:
-        users:
+        #users:
-          tenants: 1
+          #tenants: 1
-          users_per_tenant: 1
+          #users_per_tenant: 1
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  KeystoneBasic.create_and_list_users:
+  #KeystoneBasic.create_and_list_users:
-    -
+    #-
-      args: {}
+      #args: {}
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 10
+        #concurrency: 10
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  KeystoneBasic.create_tenant:
+  #KeystoneBasic.create_tenant:
-    -
+    #-
-      args: {}
+      #args: {}
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 10
+        #concurrency: 10
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  KeystoneBasic.create_tenant_with_users:
+  #KeystoneBasic.create_tenant_with_users:
-    -
+    #-
-      args:
+      #args:
-        users_per_tenant: 10
+        #users_per_tenant: 10
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 10
+        #concurrency: 10
-      context:
+      #context:
-        users:
+        #users:
-          tenants: 3
+          #tenants: 3
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  KeystoneBasic.create_user_update_password:
+  #KeystoneBasic.create_user_update_password:
-    -
+    #-
-      args: {}
+      #args: {}
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 5
+        #concurrency: 5
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  KeystoneBasic.create_update_and_delete_tenant:
+  #KeystoneBasic.create_update_and_delete_tenant:
-    -
+    #-
-      args: {}
+      #args: {}
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 5
+        #concurrency: 5
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  KeystoneBasic.create_delete_user:
+  #KeystoneBasic.create_delete_user:
-    -
+    #-
-      args: {}
+      #args: {}
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 10
+        #concurrency: 10
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  KeystoneBasic.create_and_delete_service:
+  #KeystoneBasic.create_and_delete_service:
-    -
+    #-
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 5
+        #concurrency: 5
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  KeystoneBasic.create_and_list_services:
+  #KeystoneBasic.create_and_list_services:
-    -
+    #-
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 5
+        #concurrency: 5
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  KeystoneBasic.create_and_list_ec2credentials:
+  #KeystoneBasic.create_and_list_ec2credentials:
-    -
+    #-
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 5
+        #concurrency: 5
-      context:
+      #context:
-        users:
+        #users:
-          tenants: 2
+          #tenants: 2
-          users_per_tenant: 2
+          #users_per_tenant: 2
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  KeystoneBasic.create_and_delete_ec2credential:
+  #KeystoneBasic.create_and_delete_ec2credential:
-    -
+    #-
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 5
+        #concurrency: 5
-      context:
+      #context:
-        users:
+        #users:
-          tenants: 2
+          #tenants: 2
-          users_per_tenant: 2
+          #users_per_tenant: 2
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  CeilometerEvents.create_user_and_get_event:
+  #CeilometerEvents.create_user_and_get_event:
-    -
+    #-
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 10
+        #concurrency: 10
-      context:
+      #context:
-        users:
+        #users:
-          tenants: 2
+          #tenants: 2
-          users_per_tenant: 2
+          #users_per_tenant: 2
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  CeilometerEvents.create_user_and_list_event_types:
+  #CeilometerEvents.create_user_and_list_event_types:
-    -
+    #-
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 10
+        #concurrency: 10
-      context:
+      #context:
-        users:
+        #users:
-          tenants: 2
+          #tenants: 2
-          users_per_tenant: 2
+          #users_per_tenant: 2
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  CeilometerEvents.create_user_and_list_events:
+  #CeilometerEvents.create_user_and_list_events:
-    -
+    #-
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 10
+        #concurrency: 10
-      context:
+      #context:
-        users:
+        #users:
-          tenants: 2
+          #tenants: 2
-          users_per_tenant: 2
+          #users_per_tenant: 2
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  CeilometerTraits.create_user_and_list_trait_descriptions:
+  #CeilometerTraits.create_user_and_list_trait_descriptions:
-    -
+    #-
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 10
+        #concurrency: 10
-      context:
+      #context:
-        users:
+        #users:
-          tenants: 2
+          #tenants: 2
-          users_per_tenant: 2
+          #users_per_tenant: 2
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  CeilometerTraits.create_user_and_list_traits:
+  #CeilometerTraits.create_user_and_list_traits:
-    -
+    #-
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 10
+        #concurrency: 10
-      context:
+      #context:
-        users:
+        #users:
-          tenants: 2
+          #tenants: 2
-          users_per_tenant: 2
+          #users_per_tenant: 2
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  CeilometerMeters.list_meters:
+  #CeilometerMeters.list_meters:
-    -
+    #-
-      runner:
+      #runner:
-         type: constant
+         #type: constant
-         times: 10
+         #times: 10
-         concurrency: 2
+         #concurrency: 2
-      context:
+      #context:
-         users:
+         #users:
-           tenants: 1
+           #tenants: 1
-           users_per_tenant: 1
+           #users_per_tenant: 1
-         ceilometer:
+         #ceilometer:
-           counter_name: "benchmark_meter"
+           #counter_name: "benchmark_meter"
-           counter_type: "gauge"
+           #counter_type: "gauge"
-           counter_unit: "%"
+           #counter_unit: "%"
-           counter_volume: 100
+           #counter_volume: 100
-           resources_per_tenant: 1
+           #resources_per_tenant: 1
-           samples_per_resource: 1
+           #samples_per_resource: 1
-           timestamp_interval: 1
+           #timestamp_interval: 1
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  CeilometerResource.list_resources:
+  #CeilometerResource.list_resources:
-    -
+    #-
-      runner:
+      #runner:
-         type: constant
+         #type: constant
-         times: 10
+         #times: 10
-         concurrency: 2
+         #concurrency: 2
-      context:
+      #context:
-         users:
+         #users:
-           tenants: 1
+           #tenants: 1
-           users_per_tenant: 1
+           #users_per_tenant: 1
-         ceilometer:
+         #ceilometer:
-           counter_name: "benchmark_meter"
+           #counter_name: "benchmark_meter"
-           counter_type: "gauge"
+           #counter_type: "gauge"
-           counter_unit: "%"
+           #counter_unit: "%"
-           counter_volume: 100
+           #counter_volume: 100
-           resources_per_tenant: 1
+           #resources_per_tenant: 1
-           samples_per_resource: 1
+           #samples_per_resource: 1
-           timestamp_interval: 1
+           #timestamp_interval: 1
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
-  CeilometerSamples.list_samples:
+  #CeilometerSamples.list_samples:
-    -
+    #-
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 10
+        #times: 10
-        concurrency: 10
+        #concurrency: 10
-      context:
+      #context:
-        users:
+        #users:
-          tenants: 1
+          #tenants: 1
-          users_per_tenant: 1
+          #users_per_tenant: 1
-        ceilometer:
+        #ceilometer:
-          counter_name: "cpu_util"
+          #counter_name: "cpu_util"
-          counter_type: "gauge"
+          #counter_type: "gauge"
-          counter_unit: "instance"
+          #counter_unit: "instance"
-          counter_volume: 1.0
+          #counter_volume: 1.0
-          resources_per_tenant: 3
+          #resources_per_tenant: 3
-          samples_per_resource: 10
+          #samples_per_resource: 10
-          timestamp_interval: 60
+          #timestamp_interval: 60
-          metadata_list:
+          #metadata_list:
-            -  status: "active"
+            #-  status: "active"
-               name: "fake_resource"
+               #name: "fake_resource"
-               deleted: "False"
+               #deleted: "False"
-               created_at: "2015-09-04T12:34:19.000000"
+               #created_at: "2015-09-04T12:34:19.000000"
-            -  status: "not_active"
+            #-  status: "not_active"
-               name: "fake_resource_1"
+               #name: "fake_resource_1"
-               deleted: "False"
+               #deleted: "False"
-               created_at: "2015-09-10T06:55:12.000000"
+               #created_at: "2015-09-10T06:55:12.000000"
-          batch_size: 5
+          #batch_size: 5
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
   Dummy.dummy:
     -
@@ -628,68 +628,68 @@
         failure_rate:
           max: 0
 
-  CeilometerStats.get_stats:
+  #CeilometerStats.get_stats:
-   -
+   #-
-     runner:
+     #runner:
-        type: constant
+        #type: constant
-        times: 10
+        #times: 10
-        concurrency: 2
+        #concurrency: 2
-     context:
+     #context:
-       users:
+       #users:
-         tenants: 2
+         #tenants: 2
-         users_per_tenant: 2
+         #users_per_tenant: 2
-       ceilometer:
+       #ceilometer:
-         counter_name: "benchmark_meter"
+         #counter_name: "benchmark_meter"
-         counter_type: "gauge"
+         #counter_type: "gauge"
-         counter_unit: "%"
+         #counter_unit: "%"
-         counter_volume: 100
+         #counter_volume: 100
-         resources_per_tenant: 100
+         #resources_per_tenant: 100
-         samples_per_resource: 100
+         #samples_per_resource: 100
-         timestamp_interval: 10
+         #timestamp_interval: 10
-         metadata_list:
+         #metadata_list:
-           -
+           #-
-             status: "active"
+             #status: "active"
-             name: "rally benchmark on"
+             #name: "rally benchmark on"
-             deleted: "false"
+             #deleted: "false"
-           -
+           #-
-             status: "terminated"
+             #status: "terminated"
-             name: "rally benchmark off"
+             #name: "rally benchmark off"
-             deleted: "true"
+             #deleted: "true"
-     args:
+     #args:
-       meter_name: "benchmark_meter"
+       #meter_name: "benchmark_meter"
-       filter_by_user_id: true
+       #filter_by_user_id: true
-       filter_by_project_id: true
+       #filter_by_project_id: true
-       filter_by_resource_id: true
+       #filter_by_resource_id: true
-       metadata_query:
+       #metadata_query:
-         status: "terminated"
+         #status: "terminated"
-       period: 300
+       #period: 300
-       groupby: "resource_id"
+       #groupby: "resource_id"
-     sla:
+     #sla:
-       failure_rate:
+       #failure_rate:
-         max: 0
+         #max: 0
 
-  CeilometerQueries.create_and_query_samples:
+  #CeilometerQueries.create_and_query_samples:
-    -
+    #-
-      args:
+      #args:
-        filter: {"=": {"counter_unit": "instance"}}
+        #filter: {"=": {"counter_unit": "instance"}}
-        orderby: !!null
+        #orderby: !!null
-        limit: 10
+        #limit: 10
-        counter_name: "cpu_util"
+        #counter_name: "cpu_util"
-        counter_type: "gauge"
+        #counter_type: "gauge"
-        counter_unit: "instance"
+        #counter_unit: "instance"
-        counter_volume: "1.0"
+        #counter_volume: "1.0"
-        resource_id: "resource_id"
+        #resource_id: "resource_id"
-      runner:
+      #runner:
-        type: "constant"
+        #type: "constant"
-        times: 20
+        #times: 20
-        concurrency: 10
+        #concurrency: 10
-      context:
+      #context:
-        users:
+        #users:
-          tenants: 1
+          #tenants: 1
-          users_per_tenant: 1
+          #users_per_tenant: 1
-      sla:
+      #sla:
-        failure_rate:
+        #failure_rate:
-          max: 0
+          #max: 0
 
   Authenticate.keystone:
     -

rally/verification/tempest/config.py

@@ -131,10 +131,18 @@ class TempestConfig(utils.RandomNameGeneratorMixin):
                         "HTTP error code %d.") % response.status_code
             raise exceptions.TempestConfigCreationFailure(msg)
 
-    def _get_service_url(self, service_type):
+    def _get_service_url(self, service_name):
-        for service in self.keystone.auth_ref["serviceCatalog"]:
+        s_type = self._get_service_type_by_service_name(service_name)
-            if self.clients.services().get(service["type"]) == service_type:
+        if hasattr(self.keystone, "service_catalog"):
-                return service["endpoints"][0]["publicURL"]
+            available_endpoints = self.keystone.service_catalog.get_endpoints()
+            service_endpoints = available_endpoints.get(s_type, [])
+            for endpoint in service_endpoints:
+                if endpoint["interface"] == "public":
+                    return endpoint["url"]
+        else:
+            for service in self.keystone.auth_ref["serviceCatalog"]:
+                if service["type"] == s_type:
+                    return service["endpoints"][0]["publicURL"]
 
     def _get_service_type_by_service_name(self, service_name):
         for s_type, s_name in six.iteritems(self.clients.services()):

tests/ci/osresources.py

@@ -78,7 +78,9 @@ class Keystone(ResourceManager):
         return self.client.users.list()
 
     def list_tenants(self):
-        return self.client.tenants.list()
+        if hasattr(self.client, "projects"):
+            return self.client.projects.list()  # V3
+        return self.client.tenants.list()  # V2
 
     def list_roles(self):
         return self.client.roles.list()

tests/unit/verification/test_config.py

@@ -82,7 +82,8 @@ class TempestConfigTestCase(test.TestCase):
         self.assertRaises(exceptions.TempestConfigCreationFailure,
                           self.tempest_conf._download_cirros_image)
 
-    def test__get_service_url(self):
+    def test__get_service_url_keystone_v2(self):
+        self.tempest_conf.keystone = mock.MagicMock(spec=["auth_ref"])
         self.tempest_conf.keystone.auth_ref = {
             "serviceCatalog": [
                 {
@@ -97,6 +98,24 @@ class TempestConfigTestCase(test.TestCase):
         self.assertEqual(
             self.tempest_conf._get_service_url("test_service"), "test_url")
 
+    def test__get_service_url_keystone_v3(self):
+
+        mock_catalog = mock.MagicMock()
+        mock_catalog.get_endpoints.return_value = {
+            "test_service_type": [
+                {
+                    "interface": "public",
+                    "url": "test_url"
+                }
+            ]
+        }
+
+        self.tempest_conf.keystone.service_catalog = mock_catalog
+        self.tempest_conf.clients.services.return_value = {
+            "test_service_type": "test_service"}
+        self.assertEqual(
+            self.tempest_conf._get_service_url("test_service"), "test_url")
+
     @mock.patch("rally.verification.tempest.config."
                 "TempestConfig._get_service_url", return_value="test_url")
     def test__configure_boto(self, mock__get_service_url):