x
/
kloudbuster
Code Issues Proposed changes

Add support for keystone v3 

Change-Id: I9e37f08f151468bfb49d259649213d11e81575b0
This commit is contained in:
Yichen Wang 2016-09-28 15:34:28 -07:00
parent 7a76c470fa
commit 880807e455
5 changed files with 150 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
kloudbuster/credentials.py
View File

@ -30,35 +30,23 @@ class Credentials(object):
dct['username'] = self.rc_username dct['username'] = self.rc_username
dct['password'] = self.rc_password dct['password'] = self.rc_password
dct['auth_url'] = self.rc_auth_url dct['auth_url'] = self.rc_auth_url
dct['tenant_name'] = self.rc_tenant_name if self.rc_identity_api_version == 3:
dct['cacert'] = self.rc_cacert dct['project_name'] = self.rc_project_name
dct['ca_cert'] = self.rc_cacert dct['project_domain_id'] = self.rc_project_domain_id
return dct dct['user_domain_id'] = self.rc_user_domain_id
else:
def get_nova_credentials(self): dct['tenant_name'] = self.rc_tenant_name
dct = {}
dct['username'] = self.rc_username
dct['api_key'] = self.rc_password
dct['auth_url'] = self.rc_auth_url
dct['project_id'] = self.rc_tenant_name
dct['cacert'] = self.rc_cacert
return dct
def get_nova_credentials_v2(self):
dct = self.get_nova_credentials()
dct['version'] = 2
return dct return dct
def _init_with_openrc_(self, openrc_contents): def _init_with_openrc_(self, openrc_contents):
export_re = re.compile('export OS_([A-Z_]*)="?(.*)') export_re = re.compile('export OS_([A-Z_]*)="?(.*)')
for line in openrc_contents.splitlines(): for line in openrc_contents.splitlines():
line = line.strip() line = line.strip()
mstr = export_re.match(line) mstr = export_re.match(line.strip())
if mstr: if mstr:
# get rid of posible trailing double quote # get rif of posible trailing double quote
# the first one was removed by the re # the first one was removed by the re
name = mstr.group(1) name, value = mstr.group(1), mstr.group(2)
value = mstr.group(2)
if value.endswith('"'): if value.endswith('"'):
value = value[:-1] value = value[:-1]
# get rid of password assignment # get rid of password assignment
@ -67,6 +55,11 @@ class Credentials(object):
# export OS_PASSWORD=$OS_PASSWORD_INPUT # export OS_PASSWORD=$OS_PASSWORD_INPUT
if value.startswith('$'): if value.startswith('$'):
continue continue
# Check if api version is provided
# Default is keystone v2
if name == 'IDENTITY_API_VERSION':
self.rc_identity_api_version = int(value)
# now match against wanted variable names # now match against wanted variable names
if name == 'USERNAME': if name == 'USERNAME':
self.rc_username = value self.rc_username = value
@ -74,10 +67,18 @@ class Credentials(object):
self.rc_auth_url = value self.rc_auth_url = value
elif name == 'TENANT_NAME': elif name == 'TENANT_NAME':
self.rc_tenant_name = value self.rc_tenant_name = value
elif name == 'CACERT': elif name == "CACERT":
self.rc_cacert = value self.rc_cacert = value
elif name == 'PASSWORD': elif name == "REGION_NAME":
self.rc_region_name = value
elif name == "PASSWORD":
self.rc_password = value self.rc_password = value
elif name == "PROJECT_NAME":
self.rc_project_name = value
elif name == "PROJECT_DOMAIN_ID" or name == "PROJECT_DOMAIN_NAME":
self.rc_project_domain_id = value
elif name == "USER_DOMAIN_ID" or name == "USER_DOMAIN_NAME":
self.rc_user_domain_id = value
# Read a openrc file and take care of the password # Read a openrc file and take care of the password
# The 2 args are passed from the command line and can be None # The 2 args are passed from the command line and can be None
@ -86,7 +87,12 @@ class Credentials(object):
self.rc_username = None self.rc_username = None
self.rc_tenant_name = None self.rc_tenant_name = None
self.rc_auth_url = None self.rc_auth_url = None
self.rc_cacert = None self.rc_cacert = False
self.rc_region_name = None
self.rc_project_name = None
self.rc_project_domain_id = None
self.rc_user_domain_id = None
self.rc_identity_api_version = 2
self.openrc_contents = openrc_contents self.openrc_contents = openrc_contents
success = True success = True
@ -104,15 +110,37 @@ class Credentials(object):
# no openrc file passed - we assume the variables have been # no openrc file passed - we assume the variables have been
# sourced by the calling shell # sourced by the calling shell
# just check that they are present # just check that they are present
for varname in ['OS_USERNAME', 'OS_AUTH_URL', 'OS_TENANT_NAME']: if 'OS_IDENTITY_API_VERSION' in os.environ:
if varname not in os.environ: self.rc_identity_api_version = int(os.environ['OS_IDENTITY_API_VERSION'])
LOG.warning("%s is missing" % varname)
success = False if self.rc_identity_api_version == 2:
if success: for varname in ['OS_USERNAME', 'OS_AUTH_URL', 'OS_TENANT_NAME']:
self.rc_username = os.environ['OS_USERNAME'] if varname not in os.environ:
self.rc_auth_url = os.environ['OS_AUTH_URL'] LOG.warning('%s is missing', varname)
self.rc_tenant_name = os.environ['OS_TENANT_NAME'] success = False
self.rc_cacert = os.environ.get('OS_CACERT', None) if success:
self.rc_username = os.environ['OS_USERNAME']
self.rc_auth_url = os.environ['OS_AUTH_URL']
self.rc_tenant_name = os.environ['OS_TENANT_NAME']
if 'OS_REGION_NAME' in os.environ:
self.rc_region_name = os.environ['OS_REGION_NAME']
elif self.rc_identity_api_version == 3:
for varname in ['OS_USERNAME', 'OS_AUTH_URL', 'OS_PROJECT_NAME',
'OS_PROJECT_DOMAIN_ID', 'OS_USER_DOMAIN_ID']:
if varname not in os.environ:
LOG.warning('%s is missing', varname)
success = False
if success:
self.rc_username = os.environ['OS_USERNAME']
self.rc_auth_url = os.environ['OS_AUTH_URL']
self.rc_project_name = os.environ['OS_PROJECT_NAME']
self.rc_project_domain_id = os.environ['OS_PROJECT_DOMAIN_ID']
self.rc_user_domain_id = os.environ['OS_USER_DOMAIN_ID']
if 'OS_CACERT' in os.environ:
self.rc_cacert = os.environ['OS_CACERT']
# always override with CLI argument if provided # always override with CLI argument if provided
if pwd: if pwd:

64
kloudbuster/force_cleanup.py
View File

@ -56,8 +56,10 @@ import time
# openstack python clients # openstack python clients
import cinderclient import cinderclient
import keystoneauth1 from keystoneclient.auth.identity import v2 as keystone_v2
from keystoneclient.v2_0 import client as keystoneclient from keystoneclient.auth.identity import v3 as keystone_v3
from keystoneclient import client as keystoneclient
from keystoneclient import session
import neutronclient import neutronclient
from novaclient.exceptions import NotFound from novaclient.exceptions import NotFound
from tabulate import tabulate from tabulate import tabulate
@ -137,12 +139,12 @@ class AbstractCleaner(object):
pass pass
class StorageCleaner(AbstractCleaner): class StorageCleaner(AbstractCleaner):
def __init__(self, creds, resources, dryrun): def __init__(self, sess, resources, dryrun):
from cinderclient.v2 import client as cclient from cinderclient import client as cclient
from novaclient.client import Client as nclient from novaclient import client as nclient
creden_nova = creds.get_nova_credentials_v2()
self.nova = nclient(endpoint_type='publicURL', **creden_nova) self.nova = nclient.Client('2', endpoint_type='publicURL', session=sess)
self.cinder = cclient.Client(endpoint_type='publicURL', **creden_nova) self.cinder = cclient.Client('2', endpoint_type='publicURL', session=sess)
res_desc = {'volumes': [self.cinder.volumes.list, {"all_tenants": 1}]} res_desc = {'volumes': [self.cinder.volumes.list, {"all_tenants": 1}]}
super(StorageCleaner, self).__init__('Storage', res_desc, resources, dryrun) super(StorageCleaner, self).__init__('Storage', res_desc, resources, dryrun)
@ -218,13 +220,11 @@ class StorageCleaner(AbstractCleaner):
pass pass
class ComputeCleaner(AbstractCleaner): class ComputeCleaner(AbstractCleaner):
def __init__(self, creds, resources, dryrun): def __init__(self, sess, resources, dryrun):
from neutronclient.v2_0 import client as nclient from neutronclient.neutron import client as nclient
from novaclient.client import Client as novaclient from novaclient import client as novaclient
creden = creds.get_credentials() self.neutron_client = nclient.Client('2.0', endpoint_type='publicURL', session=sess)
creden_nova = creds.get_nova_credentials_v2() self.nova_client = novaclient.Client('2', endpoint_type='publicURL', session=sess)
self.neutron_client = nclient.Client(endpoint_type='publicURL', **creden)
self.nova_client = novaclient(endpoint_type='publicURL', **creden_nova)
res_desc = { res_desc = {
'instances': [self.nova_client.servers.list, {"all_tenants": 1}], 'instances': [self.nova_client.servers.list, {"all_tenants": 1}],
'flavors': [self.nova_client.flavors.list], 'flavors': [self.nova_client.flavors.list],
@ -321,10 +321,9 @@ class ComputeCleaner(AbstractCleaner):
class NetworkCleaner(AbstractCleaner): class NetworkCleaner(AbstractCleaner):
def __init__(self, creds, resources, dryrun): def __init__(self, sess, resources, dryrun):
from neutronclient.v2_0 import client as nclient from neutronclient.neutron import client as nclient
creden = creds.get_credentials() self.neutron = nclient.Client('2.0', endpoint_type='publicURL', session=sess)
self.neutron = nclient.Client(endpoint_type='publicURL', **creden)
# because the response has an extra level of indirection # because the response has an extra level of indirection
# we need to extract it to present the list of network or router objects # we need to extract it to present the list of network or router objects
@ -430,12 +429,13 @@ class NetworkCleaner(AbstractCleaner):
class KeystoneCleaner(AbstractCleaner): class KeystoneCleaner(AbstractCleaner):
def __init__(self, creds, resources, dryrun): def __init__(self, sess, resources, dryrun):
crd = creds.get_credentials() self.keystone = keystoneclient.Client(endpoint_type='publicURL', session=sess)
self.keystone = keystoneclient.Client(endpoint_type='publicURL', **crd) self.tenant_api = self.keystone.tenants \
if self.keystone.version == 'v2.0' else self.keystone.projects
res_desc = { res_desc = {
'users': [self.keystone.users.list], 'users': [self.keystone.users.list],
'tenants': [self.keystone.tenants.list] 'tenants': [self.tenant_api.list]
} }
super(KeystoneCleaner, self).__init__('Keystone', res_desc, resources, dryrun) super(KeystoneCleaner, self).__init__('Keystone', res_desc, resources, dryrun)
@ -449,7 +449,7 @@ class KeystoneCleaner(AbstractCleaner):
else: else:
self.keystone.users.delete(id) self.keystone.users.delete(id)
self.report_deletion('USER', name) self.report_deletion('USER', name)
except keystoneauth1.exceptions.http.NotFound: except keystoneclient.auth.exceptions.http.NotFound:
self.report_not_found('USER', name) self.report_not_found('USER', name)
except KeyError: except KeyError:
pass pass
@ -458,21 +458,27 @@ class KeystoneCleaner(AbstractCleaner):
for id, name in self.resources['tenants'].iteritems(): for id, name in self.resources['tenants'].iteritems():
try: try:
if self.dryrun: if self.dryrun:
self.keystone.tenants.get(id) self.tenant_api.get(id)
else: else:
self.keystone.tenants.delete(id) self.tenant_api.delete(id)
self.report_deletion('TENANT', name) self.report_deletion('TENANT', name)
except keystoneauth1.exceptions.http.NotFound: except keystoneclient.auth.exceptions.http.NotFound:
self.report_not_found('TENANT', name) self.report_not_found('TENANT', name)
except KeyError: except KeyError:
pass pass
class KbCleaners(object): class KbCleaners(object):
def __init__(self, cred, resources, dryrun): def __init__(self, creds_obj, resources, dryrun):
self.cleaners = [] self.cleaners = []
creds = creds_obj.get_credentials()
if creds_obj.rc_identity_api_version == 3:
auth = keystone_v3.Password(**creds)
else:
auth = keystone_v2.Password(**creds)
sess = session.Session(auth=auth, verify=creds_obj.rc_cacert)
for cleaner_type in [StorageCleaner, ComputeCleaner, NetworkCleaner, KeystoneCleaner]: for cleaner_type in [StorageCleaner, ComputeCleaner, NetworkCleaner, KeystoneCleaner]:
self.cleaners.append(cleaner_type(cred, resources, dryrun)) self.cleaners.append(cleaner_type(sess, resources, dryrun))
def show_resources(self): def show_resources(self):
table = [["Type", "Name", "UUID"]] table = [["Type", "Name", "UUID"]]

56
kloudbuster/kloudbuster.py
View File

@ -25,8 +25,8 @@ import webbrowser
import base_compute import base_compute
import base_network import base_network
from glanceclient import client as glanceclient
import glanceclient.exc as glance_exception import glanceclient.exc as glance_exception
from glanceclient.v1 import client as glanceclient
from kb_config import KBConfig from kb_config import KBConfig
from kb_res_logger import KBResLogger from kb_res_logger import KBResLogger
from kb_runner_base import KBException from kb_runner_base import KBException
@ -35,9 +35,12 @@ from kb_runner_multicast import KBRunner_Multicast
from kb_runner_storage import KBRunner_Storage from kb_runner_storage import KBRunner_Storage
from kb_scheduler import KBScheduler from kb_scheduler import KBScheduler
import kb_vm_agent import kb_vm_agent
from keystoneclient.v2_0 import client as keystoneclient from keystoneclient.auth.identity import v2 as keystone_v2
from keystoneclient.auth.identity import v3 as keystone_v3
from keystoneclient import client as keystoneclient
from keystoneclient import session
import log as logging import log as logging
from novaclient.client import Client as novaclient from novaclient import client as novaclient
from oslo_config import cfg from oslo_config import cfg
import pbr.version import pbr.version
from pkg_resources import resource_filename from pkg_resources import resource_filename
@ -53,12 +56,18 @@ __version__ = pbr.version.VersionInfo('kloudbuster').version_string_with_vcs()
class KBVMCreationException(Exception): class KBVMCreationException(Exception):
pass pass
def create_keystone_client(creds): def create_auth_session(creds_obj):
""" """
Return the keystone client and auth URL given a credential Return the authenticated session
""" """
creds = creds.get_credentials() creds = creds_obj.get_credentials()
return keystoneclient.Client(endpoint_type='publicURL', **creds) if creds_obj.rc_identity_api_version == 3:
auth = keystone_v3.Password(**creds)
else:
auth = keystone_v2.Password(**creds)
sess = session.Session(auth=auth, verify=creds_obj.rc_cacert)
return sess
class Kloud(object): class Kloud(object):
def __init__(self, scale_cfg, cred, reusing_tenants, def __init__(self, scale_cfg, cred, reusing_tenants,
@ -69,8 +78,7 @@ class Kloud(object):
self.reusing_tenants = reusing_tenants self.reusing_tenants = reusing_tenants
self.storage_mode = storage_mode self.storage_mode = storage_mode
self.multicast_mode = multicast_mode self.multicast_mode = multicast_mode
self.cred = cred self.osclient_session = create_auth_session(cred)
self.keystone = create_keystone_client(cred)
self.flavor_to_use = None self.flavor_to_use = None
self.vm_up_count = 0 self.vm_up_count = 0
self.res_logger = KBResLogger() self.res_logger = KBResLogger()
@ -85,6 +93,9 @@ class Kloud(object):
if scale_cfg['availability_zone'] else None if scale_cfg['availability_zone'] else None
self.exc_info = None self.exc_info = None
self.keystone = keystoneclient.Client(session=self.osclient_session,
endpoint_type='publicURL')
LOG.info("Creating kloud: " + self.prefix) LOG.info("Creating kloud: " + self.prefix)
if self.placement_az: if self.placement_az:
LOG.info('%s Availability Zone: %s' % (self.name, self.placement_az)) LOG.info('%s Availability Zone: %s' % (self.name, self.placement_az))
@ -288,9 +299,9 @@ class KloudBuster(object):
def get_hypervisor_list(self, cred): def get_hypervisor_list(self, cred):
ret_list = [] ret_list = []
creden_nova = cred.get_nova_credentials_v2() sess = create_auth_session(cred)
nova_client = novaclient(endpoint_type='publicURL', nova_client = novaclient('2', endpoint_type='publicURL',
http_log_debug=True, **creden_nova) http_log_debug=True, session=sess)
for hypervisor in nova_client.hypervisors.list(): for hypervisor in nova_client.hypervisors.list():
if vars(hypervisor)['status'] == 'enabled': if vars(hypervisor)['status'] == 'enabled':
ret_list.append(vars(hypervisor)['hypervisor_hostname']) ret_list.append(vars(hypervisor)['hypervisor_hostname'])
@ -299,9 +310,9 @@ class KloudBuster(object):
def get_az_list(self, cred): def get_az_list(self, cred):
ret_list = [] ret_list = []
creden_nova = cred.get_nova_credentials_v2() sess = create_auth_session(cred)
nova_client = novaclient(endpoint_type='publicURL', nova_client = novaclient('2', endpoint_type='publicURL',
http_log_debug=True, **creden_nova) http_log_debug=True, session=sess)
for az in nova_client.availability_zones.list(): for az in nova_client.availability_zones.list():
zoneName = vars(az)['zoneName'] zoneName = vars(az)['zoneName']
isAvail = vars(az)['zoneState']['available'] isAvail = vars(az)['zoneState']['available']
@ -312,21 +323,14 @@ class KloudBuster(object):
def check_and_upload_images(self, retry_count=150): def check_and_upload_images(self, retry_count=150):
retry = 0 retry = 0
creds_list = [ creds_list = [create_auth_session(self.server_cred),
{'keystone': create_keystone_client(self.server_cred), 'cred': self.server_cred}, create_auth_session(self.client_cred)]
{'keystone': create_keystone_client(self.client_cred), 'cred': self.client_cred}
]
creds_dict = dict(zip(['Server kloud', 'Client kloud'], creds_list)) creds_dict = dict(zip(['Server kloud', 'Client kloud'], creds_list))
img_name_dict = dict(zip(['Server kloud', 'Client kloud'], img_name_dict = dict(zip(['Server kloud', 'Client kloud'],
[self.server_cfg.image_name, self.client_cfg.image_name])) [self.server_cfg.image_name, self.client_cfg.image_name]))
for kloud, creds in creds_dict.items(): for kloud, sess in creds_dict.items():
keystone = creds['keystone'] glance_client = glanceclient.Client('1', session=sess)
cacert = creds['cred'].get_credentials()['cacert']
glance_endpoint = keystone.service_catalog.url_for(
service_type='image', endpoint_type='publicURL')
glance_client = glanceclient.Client(
glance_endpoint, token=keystone.auth_token, cacert=cacert)
try: try:
# Search for the image # Search for the image
img = glance_client.images.list(filters={'name': img_name_dict[kloud]}).next() img = glance_client.images.list(filters={'name': img_name_dict[kloud]}).next()

13
kloudbuster/tenant.py
View File

@ -36,6 +36,8 @@ class Tenant(object):
self.kloud = kloud self.kloud = kloud
self.res_logger = kloud.res_logger self.res_logger = kloud.res_logger
self.tenant_name = tenant_name self.tenant_name = tenant_name
self.tenant_api = self.kloud.keystone.tenants \
if self.kloud.keystone.version == 'v2.0' else self.kloud.keystone.projects
if not self.kloud.reusing_tenants: if not self.kloud.reusing_tenants:
self.tenant_object = self._get_tenant() self.tenant_object = self._get_tenant()
self.tenant_id = self.tenant_object.id self.tenant_id = self.tenant_object.id
@ -58,9 +60,10 @@ class Tenant(object):
try: try:
LOG.info("Creating tenant: " + self.tenant_name) LOG.info("Creating tenant: " + self.tenant_name)
tenant_object = \ tenant_object = \
self.kloud.keystone.tenants.create(tenant_name=self.tenant_name, self.tenant_api.create(self.tenant_name,
description="KloudBuster tenant", domain="default",
enabled=True) description="KloudBuster tenant",
enabled=True)
return tenant_object return tenant_object
except keystone_exception.Conflict as exc: except keystone_exception.Conflict as exc:
# ost likely the entry already exists: # ost likely the entry already exists:
@ -68,7 +71,7 @@ class Tenant(object):
if exc.http_status != 409: if exc.http_status != 409:
raise exc raise exc
LOG.info("Tenant %s already present, reusing it" % self.tenant_name) LOG.info("Tenant %s already present, reusing it" % self.tenant_name)
return self.kloud.keystone.tenants.find(name=self.tenant_name) return self.tenant_api.find(name=self.tenant_name)
# Should never come here # Should never come here
raise Exception() raise Exception()
@ -122,6 +125,6 @@ class Tenant(object):
if not self.reusing_users: if not self.reusing_users:
# Delete the tenant (self) # Delete the tenant (self)
self.kloud.keystone.tenants.delete(self.tenant_id) self.tenant_api.delete(self.tenant_id)
return flag return flag

38
kloudbuster/users.py
View File

@ -16,10 +16,10 @@ import sys
import base_compute import base_compute
import base_network import base_network
from cinderclient.v2 import client as cinderclient from cinderclient import client as cinderclient
from keystoneclient import exceptions as keystone_exception from keystoneclient import exceptions as keystone_exception
import log as logging import log as logging
from neutronclient.v2_0 import client as neutronclient from neutronclient.neutron import client as neutronclient
from novaclient import client as novaclient from novaclient import client as novaclient
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
@ -66,9 +66,12 @@ class User(object):
if not self.tenant.reusing_users: if not self.tenant.reusing_users:
self.user = self._get_user() self.user = self._get_user()
current_role = self.tenant.kloud.keystone.roles.find(name=user_role) current_role = self.tenant.kloud.keystone.roles.find(name=user_role)
self.tenant.kloud.keystone.roles.add_user_role(self.user, if self.tenant.kloud.keystone.version == 'v2.0':
current_role, self.tenant.kloud.keystone.roles.add_user_role(
tenant.tenant_id) self.user, current_role, tenant.tenant_id)
else:
self.tenant.kloud.keystone.roles.grant(
current_role, user=self.user, project=tenant.tenant_id)
else: else:
# Only admin can retrive the object via Keystone API # Only admin can retrive the object via Keystone API
self.user = None self.user = None
@ -190,24 +193,15 @@ class User(object):
1. Creates the routers 1. Creates the routers
2. Creates the neutron and nova client objects 2. Creates the neutron and nova client objects
""" """
# Create a new neutron client for this User with correct credentials session = self.tenant.kloud.osclient_session
creden = self.tenant.kloud.cred.get_credentials()
creden['username'] = self.user_name
creden['password'] = self.password
creden['tenant_name'] = self.tenant.tenant_name
# Create the neutron client to be used for all operations # Create nova/neutron/cinder clients to be used for all operations
self.neutron_client = neutronclient.Client(endpoint_type='publicURL', **creden) self.neutron_client = neutronclient.Client('2.0', endpoint_type='publicURL',
session=session)
# Create a new nova and cinder client for this User with correct credentials self.nova_client = novaclient.Client('2', endpoint_type='publicURL',
creden_nova = self.tenant.kloud.cred.get_nova_credentials_v2() http_log_debug=True, session=session)
creden_nova['username'] = self.user_name self.cinder_client = cinderclient.Client('2', endpoint_type='publicURL',
creden_nova['api_key'] = self.password session=session)
creden_nova['project_id'] = self.tenant.tenant_name
self.nova_client = novaclient.Client(endpoint_type='publicURL',
http_log_debug=True, **creden_nova)
self.cinder_client = cinderclient.Client(endpoint_type='publicURL', **creden_nova)
if self.tenant.kloud.reusing_tenants: if self.tenant.kloud.reusing_tenants:
self.check_resources_quota() self.check_resources_quota()