Enhance the supports for tenant/user reusing

When running under tenant/user reusing mode:
1. Adding features to automatically pick the flavor;
2. Adding features to check quotas before testing;
3. Merge the server and client tenant into one;

Change-Id: Ibdda7624174056d7770de02ed0813cad6db2ac05

scale/base_compute.py

@@ -217,6 +217,9 @@ class Flavor(object):
     def __init__(self, novaclient):
         self.novaclient = novaclient
 
+    def list(self):
+        return self.novaclient.flavors.list()
+
     def create_flavor(self, name, ram, vcpus, disk, override=False):
         # Creating flavors
         if override:
@@ -236,6 +239,9 @@ class NovaQuota(object):
         self.novaclient = novaclient
         self.tenant_id = tenant_id
 
+    def get(self):
+        return self.novaclient.quotas.get(self.tenant_id).__dict__
+
     def update_quota(self, **kwargs):
         self.novaclient.quotas.update(self.tenant_id, **kwargs)
 
@@ -245,5 +251,8 @@ class CinderQuota(object):
         self.cinderclient = cinderclient
         self.tenant_id = tenant_id
 
+    def get(self):
+        return self.cinderclient.quotas.get(self.tenant_id).__dict__
+
     def update_quota(self, **kwargs):
         self.cinderclient.quotas.update(self.tenant_id, **kwargs)

scale/base_network.py

@@ -23,12 +23,14 @@ from neutronclient.common.exceptions import NetworkInUseClient
 
 LOG = logging.getLogger(__name__)
 
-
 # Global CIDR shared by all objects of this class
 # Enables each network to get a unique CIDR
 START_CIDR = "10.0.0.0/16"
 cidr = START_CIDR
 
+class KBGetExtNetException(Exception):
+    pass
+
 def create_floating_ip(neutron_client, ext_net):
     """
     Function that creates a floating ip and returns it
@@ -63,8 +65,8 @@ def find_external_network(neutron_client):
         if network['router:external']:
             return network
 
-    LOG.error("No external network found!!!")
+    LOG.error("No external network is found.")
-    return None
+    raise KBGetExtNetException()
 
 
 class BaseNetwork(object):
@@ -375,6 +377,9 @@ class NeutronQuota(object):
         self.neutronclient = neutronclient
         self.tenant_id = tenant_id
 
+    def get(self):
+        return self.neutronclient.show_quota(self.tenant_id)['quota']
+
     def update_quota(self, quotas):
         body = {
             'quota': quotas

scale/cfg.tenants.yaml

@@ -7,22 +7,20 @@
 #
 # Settings in this file has higher priority than user configs. It determines
 # the final count of tenants and useres that KloudBuster will use.
+#
+# If running under tenant/user reusing mode, KloudBuster will use *only* one
+# tenant to hold the resources for both server cloud and client cloud.
+#
+# NOTE:
+# (1) For now, we only support one user per tenant;
+# (2) Under tenant/user resuing mode, all resources will be sitting under
+#     the same tenant, so there will be fixed *ONLY* one user for holding
+#     client side resources;
 
-
+tenant_name: demo_tenant
-# Sections for listing the tenant and user lists for server cloud.
+server_user:
-# Note: For now we only support 1 user per tenant
+    - username: demo_user_1
-server:
+      password: demo_user_1
-    - name: ts_1
+client_user:
-      user:
+    username: demo_user_2
-          - username: tsu_1
+    password: demo_user_2
-            password: tsu_1
-    - name: ts_2
-      user:
-          - username: tsu_2
-            password: tsu_2
-
-client:
-    - name: tc_1
-      user:
-          - username: tcu_1
-            password: tcu_1

scale/kb_config.py

@@ -154,8 +154,8 @@ class KBConfig(object):
         if CONF.tenants_list:
             self.tenants_list = configure.Configuration.from_file(CONF.tenants_list).configure()
             try:
-                self.config_scale['number_tenants'] = len(self.tenants_list['server'])
+                self.config_scale['number_tenants'] = 1
-                self.config_scale['users_per_tenant'] = len(self.tenants_list['server'][0]['user'])
+                self.config_scale['users_per_tenant'] = len(self.tenants_list['server_user'])
             except Exception as e:
                 LOG.error('Cannot parse the count of tenant/user from the config file.')
                 raise KBConfigParseException(e.message)

scale/kloudbuster.py

@@ -65,8 +65,8 @@ def check_and_upload_images(cred, cred_testing, server_img_name, client_img_name
             if img['name'] == img_name_dict[kloud]:
                 img_found = True
                 break
-        if img.visibility != 'public' and CONF.tenants_list:
+        if img_found and img.visibility != 'public' and CONF.tenants_list:
-            LOG.error("Image must be public when running in reusing mode.")
+            LOG.error("Image must be public when running in tenant/user reusing mode.")
             sys.exit(1)
 
         if not img_found:
@@ -236,9 +236,13 @@ class KloudBuster(object):
         else:
             self.topology = topology
         if tenants_list:
-            self.tenants_list = tenants_list
+            self.tenants_list = {}
-            LOG.warn("REUSING MODE: The quota will not adjust automatically.")
+            self.tenants_list['server'] =\
-            LOG.warn("REUSING MODE: The flavor configs will be ignored, and m1.small is used.")
+                [{'name': tenants_list['tenant_name'], 'user': tenants_list['server_user']}]
+            self.tenants_list['client'] =\
+                [{'name': tenants_list['tenant_name'], 'user': [tenants_list['client_user']]}]
+            LOG.warn("REUSING MODE: The quotas will not be adjusted automatically.")
+            LOG.warn("REUSING MODE: The flavor configs will be ignored.")
         else:
             self.tenants_list = {'server': None, 'client': None}
         # TODO(check on same auth_url instead)
@@ -427,7 +431,7 @@ class KloudBuster(object):
             server_quota['floatingip'] = server_quota['router']
             server_quota['port'] = total_vm + 2 * server_quota['network'] + server_quota['router']
         server_quota['security_group'] = server_quota['network'] + 1
-        server_quota['security_group_rule'] = server_quota['security_group'] * 100
+        server_quota['security_group_rule'] = server_quota['security_group'] * 10
 
         client_quota = {}
         total_vm = total_vm * self.server_cfg['number_tenants']
@@ -454,7 +458,7 @@ class KloudBuster(object):
             # cloud, and each one takes up 1 port on client side.
             client_quota['port'] = client_quota['port'] + server_quota['router']
         client_quota['security_group'] = client_quota['network'] + 1
-        client_quota['security_group_rule'] = client_quota['security_group'] * 100
+        client_quota['security_group_rule'] = client_quota['security_group'] * 10
 
         return [server_quota, client_quota]
 

scale/tenant.py

@@ -34,23 +34,28 @@ class Tenant(object):
         Stores the shared network in case of testing and
         tested cloud being on same cloud
         """
-        self.tenant_name = tenant_name
         self.kloud = kloud
-        self.tenant_object = self._get_tenant()
+        self.tenant_name = tenant_name
-        self.tenant_id = self.tenant_object.id
+        if not self.kloud.reusing_tenants:
+            self.tenant_object = self._get_tenant()
+            self.tenant_id = self.tenant_object.id
+        else:
+            LOG.info("Using tenant: " + self.tenant_name)
+            # Only admin can retrive the object via Keystone API
+            self.tenant_object = None
+            try:
+                # Try to see if we have the admin access to retrive the tenant id using
+                # tenant name directly from keystone
+                self.tenant_id = self.kloud.keystone.tenants.find(name=self.tenant_name).id
+            except Exception:
+                self.tenant_id = self.kloud.keystone.tenant_id
+
         self.tenant_quota = tenant_quota
         self.reusing_users = reusing_users
         # Contains a list of user instance objects
         self.user_list = []
 
     def _get_tenant(self):
-        if self.kloud.reusing_tenants:
-            LOG.info("Using tenant: " + self.tenant_name)
-            tenant_list = self.kloud.keystone.tenants.list()
-            for tenant in tenant_list:
-                if tenant.name == self.tenant_name:
-                    return tenant
-            raise Exception("Tenant not found")
 
         '''
         Create or reuse a tenant object of a given name
@@ -68,15 +73,10 @@ class Tenant(object):
             if exc.http_status != 409:
                 raise exc
             LOG.info("Tenant %s already present, reusing it" % self.tenant_name)
-            # It is a hassle to find a tenant by name as the only way seems to retrieve
+            return self.kloud.keystone.tenants.find(name=self.tenant_name)
-            # the list of all tenants which can be very large
-            tenant_list = self.kloud.keystone.tenants.list()
-            for tenant in tenant_list:
-                if tenant.name == self.tenant_name:
-                    return tenant
 
         # Should never come here
-        raise Exception("Tenant not found")
+        raise Exception()
 
     def create_resources(self):
         """
@@ -87,7 +87,7 @@ class Tenant(object):
             for user_info in self.reusing_users:
                 user_name = user_info['username']
                 password = user_info['password']
-                user_instance = users.User(user_name, password, self, None)
+                user_instance = users.User(user_name, password, self, '_member_')
                 self.user_list.append(user_instance)
         else:
             # Loop over the required number of users and create resources

scale/users.py

@@ -22,6 +22,11 @@ from novaclient.client import Client
 
 LOG = logging.getLogger(__name__)
 
+class KBFlavorCheckException(Exception):
+    pass
+
+class KBQuotaCheckException(Exception):
+    pass
 
 class User(object):
     """
@@ -45,7 +50,6 @@ class User(object):
         self.nova_client = None
         self.neutron_client = None
         self.cinder_client = None
-        self.user = self._get_user()
         # Each user is associated to 1 key pair at most
         self.key_pair = None
         self.key_name = None
@@ -56,15 +60,17 @@ class User(object):
         #
         # If running on top of existing tenants/users, skip
         # the step for admin role association.
-        if not self.tenant.kloud.reusing_tenants:
+        if not self.tenant.reusing_users:
-            current_role = None
+            self.user = self._get_user()
-            for role in self.tenant.kloud.keystone.roles.list():
+            current_role = self.tenant.kloud.keystone.roles.find(name=user_role)
-                if role.name == user_role:
-                    current_role = role
-                    break
             self.tenant.kloud.keystone.roles.add_user_role(self.user,
                                                            current_role,
                                                            tenant.tenant_id)
+        else:
+            # Only admin can retrive the object via Keystone API
+            self.user = None
+            LOG.info("Using user: " + self.user_name)
+
 
     def _create_user(self):
         LOG.info("Creating user: " + self.user_name)
@@ -74,14 +80,6 @@ class User(object):
                                                        tenant_id=self.tenant.tenant_id)
 
     def _get_user(self):
-        if self.tenant.reusing_users:
-            LOG.info("Using user: " + self.user_name)
-            users_list = self.tenant.kloud.keystone.users.list()
-            for user in users_list:
-                if user.name == self.user_name:
-                    return user
-            raise Exception('Cannot find stale user:' + self.user_name)
-
         '''
         Create a new user or reuse if it already exists (on a different tenant)
         delete the user and create a new one
@@ -97,19 +95,13 @@ class User(object):
             # Try to repair keystone by removing that user
             LOG.warn("User creation failed due to stale user with same name: " +
                      self.user_name)
-            # Again, trying to find a user by name is pretty inefficient as one has to list all
+            user = self.tenant.kloud.keystone.users.find(name=self.user_name)
-            # of them
+            LOG.info("Deleting stale user with name: " + self.user_name)
-            users_list = self.tenant.kloud.keystone.users.list()
+            self.tenant.kloud.keystone.users.delete(user)
-            for user in users_list:
+            return self._create_user()
-                if user.name == self.user_name:
-                    # Found it, time to delete it
-                    LOG.info("Deleting stale user with name: " + self.user_name)
-                    self.tenant.kloud.keystone.users.delete(user)
-                    user = self._create_user()
-                    return user
 
-        # Not found there is something wrong
+        # Should never come here
-        raise Exception('Cannot find stale user:' + self.user_name)
+        raise Exception()
 
     def delete_resources(self):
         LOG.info("Deleting all user resources for user %s" % self.user_name)
@@ -136,6 +128,47 @@ class User(object):
         neutron_quota = base_network.NeutronQuota(self.neutron_client, self.tenant.tenant_id)
         neutron_quota.update_quota(tenant_quota['neutron'])
 
+    def check_resources_quota(self):
+        # Flavor check
+        flavor_manager = base_compute.Flavor(self.nova_client)
+        flavor_to_use = None
+        for flavor in flavor_manager.list():
+            flavor = flavor.__dict__
+            if flavor['vcpus'] < 1 or flavor['ram'] < 1024 or flavor['disk'] < 10:
+                continue
+            flavor_to_use = flavor
+            break
+        if flavor_to_use:
+            LOG.info('Automatically selects flavor %s to instantiate VMs.' %
+                     (flavor_to_use['name']))
+        else:
+            LOG.error('Cannot find a flavor which meets the minimum '
+                      'requirements to instantiate VMs.')
+            raise KBFlavorCheckException()
+
+        # Nova/Cinder/Neutron quota check
+        tenant_id = self.tenant.tenant_id
+        meet_quota = True
+        for quota_type in ['nova', 'cinder', 'neutron']:
+            if quota_type == 'nova':
+                quota_manager = base_compute.NovaQuota(self.nova_client, tenant_id)
+            elif quota_type == 'cinder':
+                quota_manager = base_compute.CinderQuota(self.cinder_client, tenant_id)
+            else:
+                quota_manager = base_network.NeutronQuota(self.neutron_client, tenant_id)
+
+            meet_quota = True
+            quota = quota_manager.get()
+            for key, value in self.tenant.tenant_quota[quota_type].iteritems():
+                if quota[key] < value:
+                    meet_quota = False
+                    break
+
+        if not meet_quota:
+            LOG.error('%s quota is too small. Minimum requirement: %s.' %
+                      (quota_type, self.tenant.tenant_quota[quota_type]))
+            raise KBQuotaCheckException()
+
     def create_resources(self):
         """
         Creates all the User elements associated with a User
@@ -145,7 +178,7 @@ class User(object):
         # Create a new neutron client for this User with correct credentials
         creden = {}
         creden['username'] = self.user_name
-        creden['password'] = self.user_name
+        creden['password'] = self.password
         creden['auth_url'] = self.tenant.kloud.auth_url
         creden['tenant_name'] = self.tenant.tenant_name
 
@@ -155,7 +188,7 @@ class User(object):
         # Create a new nova and cinder client for this User with correct credentials
         creden_nova = {}
         creden_nova['username'] = self.user_name
-        creden_nova['api_key'] = self.user_name
+        creden_nova['api_key'] = self.password
         creden_nova['auth_url'] = self.tenant.kloud.auth_url
         creden_nova['project_id'] = self.tenant.tenant_name
         creden_nova['version'] = 2
@@ -163,7 +196,9 @@ class User(object):
         self.nova_client = Client(**creden_nova)
         self.cinder_client = cinderclient.Client(**creden_nova)
 
-        if not self.tenant.kloud.reusing_tenants:
+        if self.tenant.kloud.reusing_tenants:
+            self.check_resources_quota()
+        else:
             self.update_tenant_quota(self.tenant.tenant_quota)
 
         config_scale = self.tenant.kloud.scale_cfg