From a48fc6e7582f9fd5f759ac560b36188f06260b9f Mon Sep 17 00:00:00 2001 From: Peter Jenkins Date: Wed, 13 Jan 2016 14:06:21 +0200 Subject: [PATCH] Update aut-fix-quota tool for Juno I've ported this tool to work with Juno and thought I'd share this for all to use. We've done basic functional testing, an internal code review and used it against our production cloud. I would encourage further review and testing, but this should be usable for others. The backend changes to Juno required more changes that I would have liked and the code does throw the odd exception (handled by the code) when run against every project in our production system, but this hasn't caused any issues for us. Change-Id: I3913251ede4949149e63d1337bd92dd836f98763 --- nova/auto-fix-quota-all-tenants.sh | 2 - nova/auto-fix-quota.py | 226 +++++++++++++++-------------- 2 files changed, 115 insertions(+), 113 deletions(-) diff --git a/nova/auto-fix-quota-all-tenants.sh b/nova/auto-fix-quota-all-tenants.sh index 5f8d729..58efb2c 100755 --- a/nova/auto-fix-quota-all-tenants.sh +++ b/nova/auto-fix-quota-all-tenants.sh @@ -1,5 +1,4 @@ #!/bin/bash -source /root/scripts/stackrc echo "$(date): Tenant quota correction - started" @@ -10,4 +9,3 @@ for x in $(keystone --insecure tenant-list | awk -F' |\ done echo "$(date): Tenant quota correction - finished" - diff --git a/nova/auto-fix-quota.py b/nova/auto-fix-quota.py index 4dd0077..4f6b433 100755 --- a/nova/auto-fix-quota.py +++ b/nova/auto-fix-quota.py @@ -7,11 +7,11 @@ from nova import db from nova import config from nova import context +from nova import exception from collections import OrderedDict import argparse import prettytable - def make_table(name, *args): q = prettytable.PrettyTable(name) q.align = "c" @@ -20,24 +20,27 @@ def make_table(name, *args): def get_actual_usage(cntxt, tenant): - instance_info = db.instance_data_get_for_project(cxt, tenant) + filter_object = {'deleted': '', + 'project_id': tenant} + instances = db.instance_get_all_by_filters(cntxt, filter_object) # calculate actual usage - actual_instance_count = instance_info[0] - actual_core_count = instance_info[1] - actual_ram_count = instance_info[2] + actual_instance_count = len(instances) + actual_core_count = 0 + actual_ram_count = 0 - # actual_fixed_ips should be the same as actual_instance_count since there is no - # reason in our environment for someone to use two fixed IPs per instance - # db.fixed_ip_count_by_project(cxt,tenant) - actual_fixed_ips = actual_instance_count - actual_secgroup_count = db.security_group_count_by_project(cxt, tenant) + for instance in instances: + actual_core_count += instance['vcpus'] + actual_ram_count += instance['memory_mb'] + + actual_secgroup_count = len(db.security_group_get_by_project(cntxt, tenant)) + if actual_secgroup_count == 0: + actual_secgroup_count = 1 # Every tenant uses quota for default security group return OrderedDict(( ("actual_instance_count", actual_instance_count), ("actual_core_count", actual_core_count), ("actual_ram_count", actual_ram_count), - ("actual_fixed_ips", actual_fixed_ips), ("actual_secgroup_count", actual_secgroup_count) )) @@ -49,109 +52,98 @@ def get_incorrect_usage(cntxt, tenant): # u'instances': {'reserved': 0L, 'in_use': 0L}, # u'cores': {'reserved': 0L, 'in_use': 0L}, # 'project_id': tenant, - # u'fixed_ips': {'reserved': 0L, 'in_use': 0L}, # u'security_groups': {'reserved': 0L, 'in_use': 1L}} # # Get (instance_count, total_cores, total_ram) for project. # If instances does not exist, then this - if 'instances' in existing_usage: - # it's possible for accounts to use nothing but the system default - # group, which doesn't get tracked, so default to 1 - try: - security_groups = existing_usage["security_groups"]["in_use"] - except KeyError: - security_groups = 1 - return OrderedDict(( - ("db_instance_count", existing_usage["instances"]["in_use"]), - ("db_core_count", existing_usage["cores"]["in_use"]), - ("db_ram_count", existing_usage["ram"]["in_use"]), - ("db_fixed_ips", existing_usage["fixed_ips"]["in_use"]), - ("db_secgroup_count", security_groups) - )) - else: - print "%s get_incorrect_usage failed to find quota usage information in " \ - "database. Is this tenant in use?" % tenant - exit(0) - - -def fix_usage(cntxt, tenant, actual_table_name, incorrect_table_name): - print "\nUpdating quota usage to reflect actual usage..\n" - # Calculate differences - existing_usage = db.quota_usage_get_all_by_project(cntxt, tenant) - actual = get_actual_usage(cntxt, tenant) - - # it's possible for accounts to use nothing but the system default - # group, which doesn't get tracked, so default to 0 try: security_groups = existing_usage["security_groups"]["in_use"] - secgroup_difference = security_groups - actual["actual_secgroup_count"] except KeyError: - security_groups = None + security_groups = 1 - instance_difference = \ - existing_usage["instances"]["in_use"] - actual["actual_instance_count"] - core_difference = \ - existing_usage["cores"]["in_use"] - actual["actual_core_count"] - ram_difference = \ - existing_usage["ram"]["in_use"] - actual["actual_ram_count"] - # Actual_fixed_ips should be the same as actual_instance_count since there is no - # Reason in our environment for someone to use two fixed IPs - # existing_usage["fixed_ips"]["in_use"]-actual["actual_fixed_ips"] - fixedips_difference = \ - existing_usage["fixed_ips"]["in_use"] - actual["actual_instance_count"] - # Quota_usage_update(context, project_id, resource, **kwargs) - # Update ram. - db.quota_usage_update(cxt, - tenant, - 'ram', - in_use=existing_usage["ram"]["in_use"] - - ram_difference) - # Update instances - db.quota_usage_update(cxt, - tenant, - 'instances', - in_use=existing_usage["instances"]["in_use"] - - instance_difference) - # Update cores - db.quota_usage_update(cxt, - tenant, - 'cores', - in_use=existing_usage["cores"]["in_use"] - core_difference) - # Update fixed IPs - """ - db.quota_usage_update(cxt, - tenant, - 'fixed_ips', - in_use=existing_usage["fixed_ips"]["in_use"] - fixedips_difference) - """ + try: + instances = existing_usage["instances"]["in_use"] + except KeyError: + instances = 0 - # Update security groups - if security_groups is not None: - db.quota_usage_update(cxt, - tenant, - 'security_groups', - in_use=security_groups - secgroup_difference) + try: + cores = existing_usage["cores"]["in_use"] + except KeyError: + cores = 0 + + try: + ram = existing_usage["ram"]["in_use"] + except KeyError: + ram = 0 + + return OrderedDict(( + ("db_instance_count", instances), + ("db_core_count", cores), + ("db_ram_count", ram), + ("db_secgroup_count", security_groups) + )) + + +def fix_usage(cntxt, tenant): + + # Get per-user data for this tenant since usage is now per-user + filter_object = {'project_id': tenant} + instance_info = db.instance_get_all_by_filters(cntxt, filter_object) + + usage_by_resource = {} + #resource_types = ['instances', 'cores', 'ram', 'security_groups'] + states_to_ignore = ['error', 'deleted', 'building'] + + for instance in instance_info: + user = instance['user_id'] + # We need to build a list of users who have launched vm's even if the user + # no longer exists. We can't use keystone here. + if not usage_by_resource.has_key(user): + usage_by_resource[user] = {} # Record that this user has once used resources + if not instance['vm_state'] in states_to_ignore: + user_resource = usage_by_resource[user] + user_resource['instances'] = user_resource.get('instances', 0) + 1 + user_resource['cores'] = user_resource.get('cores', 0) + instance['vcpus'] + user_resource['ram'] = user_resource.get('ram', 0) + instance['memory_mb'] + + secgroup_list = db.security_group_get_by_project(cntxt, tenant) + for group in secgroup_list: + user = group.user_id + if not usage_by_resource.has_key(user): + usage_by_resource[user] = {} # Record that this user has once used resources + user_resource = usage_by_resource[user] + user_resource['security_groups'] = user_resource.get('security_groups', 0) + 1 + + # Correct the quota usage in the database + for user in usage_by_resource: + for resource in resource_types: + usage = usage_by_resource[user].get(resource, 0) + try: + db.quota_usage_update(cntxt, tenant, user, resource, in_use=usage) + except exception.QuotaUsageNotFound as e: + print e + print 'db.quota_usage_update(cntxt, %s, %s, %s, in_use=%s)' % \ + (tenant, user, resource, usage) + +def print_usage(cntxt, tenant): + actual_table_name = ["Actual Instances", + "Actual Cores", + "Actual RAM", + "Actual Security_Groups"] + + # these are spaced so that the Quota & DB tables match in size + incorrect_table_name = [" DB Instances ", + " DB Cores ", + " DB RAM ", + " DB Security_Groups "] print "############### Actual Usage (including non-active instances) ###############" - print make_table(actual_table_name, get_actual_usage(cxt, tenant).values()) - print "############### Corrected Database Usage ###############" - print make_table(incorrect_table_name, get_incorrect_usage(cxt, tenant).values()) - - -actual_table_name = ["Actual Instances", - "Actual Cores", - "Actual RAM", - "Actual Fixed_ips", - "Actual Security_Groups"] - -# these are spaced so that the Quota & DB tables match in size -incorrect_table_name = [" DB Instances ", - " DB Cores ", - " DB RAM ", - " DB Fixed_ips ", - " DB Security_Groups "] + print make_table(actual_table_name, get_actual_usage(cntxt, tenant).values()) + print "############### Database Usage ###############" + print make_table(incorrect_table_name, get_incorrect_usage(cntxt, tenant).values()) +resource_types = ['instances', 'cores', 'ram', 'security_groups'] config.parse_args(['filename', '--config-file', '/etc/nova/nova.conf']) # Get other arguments @@ -160,25 +152,37 @@ parser = argparse.ArgumentParser( parser.add_argument('--tenant', help='Specify tenant', required=True) parser.add_argument('-n', '--dryrun', help='Dry Run - don\'t update the database', action="store_true") +parser.add_argument('-q', '--quiet', help='Quiet mode. Only show incorrect quotas', + action="store_true") args = parser.parse_args() tenant = args.tenant # Get admin context cxt = context.get_admin_context() -print "############### Actual Usage (including non-active instances) ###############" -print make_table(actual_table_name, get_actual_usage(cxt, tenant).values()) -print "############### Database Usage ###############" -print make_table(incorrect_table_name, get_incorrect_usage(cxt, tenant).values()) - # if the actual usage & the quota tracking differ, # update quota to match reality -if get_incorrect_usage(cxt, tenant).values() == get_actual_usage(cxt, tenant).values(): - print "\n%s quota is OK" % tenant -elif args.dryrun: - print "Dry Run Mode Enabled - not correcting the quota database." +try: + actual = get_actual_usage(cxt, tenant).values() + incorrect = get_incorrect_usage(cxt, tenant).values() +except: + exit(2) + +if actual == incorrect: + if not args.quiet: + print_usage(cxt, tenant) + print "%s quota is OK" % tenant + exit(0) else: - fix_usage(cxt, tenant, actual_table_name, incorrect_table_name) + print "%s usage and database differ" % tenant + print_usage(cxt, tenant) + if args.dryrun: + print "Dry Run Mode Enabled - not correcting the quota database." + exit(1) + else: + print "Updating quota usage to reflect actual usage..\n" + fix_usage(cxt, tenant) + print_usage(cxt, tenant) # This section can replace the final if/else statement to allow prompting for # each tenant before changes happen