Fix deprecated API usage of keystone client

* Use keystoneauth1 for making session, which is shared throw all clients
* KeystoneClient is now only wrapper over session
Requirements has been updated to meet global requirements for mitaka release:
1. Bumped clients versions
2. requests has blacklisted versions due to bugs

Closes-bug: #1571611

Change-Id: Icc59761b590b76a8d3ddac9b4f219efc097447a5
(cherry picked from commit 94170e0)
This commit is contained in:
Alexey Stepanov 2016-04-18 14:36:04 +03:00
parent 6d7f77180e
commit c28d12604b
4 changed files with 223 additions and 155 deletions

View File

@ -12,21 +12,33 @@
# License for the specific language governing permissions and limitations
# under the License.
import sys
import time
from urlparse import urlparse
import traceback
from cinderclient import client as cinderclient
from cinderclient.client import Client as CinderClient
from heatclient.v1.client import Client as HeatClient
from glanceclient.v1 import Client as GlanceClient
import ironicclient.client as ironicclient
from ironicclient.client import Client as IronicClient
from keystoneauth1.exceptions import ClientException
from keystoneauth1.identity import V2Password
from keystoneauth1.session import Session as KeystoneSession
from keystoneclient.v2_0 import Client as KeystoneClient
from keystoneclient.exceptions import ClientException
from novaclient.v2 import Client as NovaClient
import neutronclient.v2_0.client as neutronclient
from novaclient.client import Client as NovaClient
from neutronclient.v2_0.client import Client as NeutronClient
from proboscis.asserts import assert_equal
import six
# pylint: disable=redefined-builtin
# noinspection PyUnresolvedReferences
from six.moves import xrange
# pylint: enable=redefined-builtin
# pylint: disable=import-error
# noinspection PyUnresolvedReferences
from six.moves import urllib
# pylint: enable=import-error
from fuelweb_test import logger as LOGGER
from fuelweb_test import logwrap as LOGWRAP
from fuelweb_test import logger
from fuelweb_test import logwrap
from fuelweb_test.settings import DISABLE_SSL
from fuelweb_test.settings import PATH_TO_CERT
from fuelweb_test.settings import VERIFY_SSL
@ -35,14 +47,16 @@ from fuelweb_test.settings import VERIFY_SSL
class Common(object):
"""Common.""" # TODO documentation
def __make_endpoint(self, endpoint):
parse = urllib.parse.urlparse(endpoint)
return parse._replace(
netloc='{}:{}'.format(
self.controller_ip, parse.port)).geturl()
def __init__(self, controller_ip, user, password, tenant):
self.controller_ip = controller_ip
def make_endpoint(endpoint):
parse = urlparse(endpoint)
return parse._replace(
netloc='{}:{}'.format(
self.controller_ip, parse.port)).geturl()
self.keystone_session = None
if DISABLE_SSL:
auth_url = 'http://{0}:5000/v2.0/'.format(self.controller_ip)
@ -53,79 +67,100 @@ class Common(object):
insecure = not VERIFY_SSL
LOGGER.debug('Auth URL is {0}'.format(auth_url))
logger.debug('Auth URL is {0}'.format(auth_url))
keystone_args = {'username': user, 'password': password,
'tenant_name': tenant, 'auth_url': auth_url,
'ca_cert': path_to_cert, 'insecure': insecure}
self.keystone = self._get_keystoneclient(**keystone_args)
self.__keystone_auth = V2Password(
auth_url=auth_url,
username=user,
password=password,
tenant_name=tenant) # TODO: in v3 project_name
token = self.keystone.auth_token
LOGGER.debug('Token is {0}'.format(token))
self.__start_keystone_session(ca_cert=path_to_cert, insecure=insecure)
neutron_endpoint = self.keystone.service_catalog.url_for(
service_type='network', endpoint_type='publicURL')
neutron_args = {'username': user, 'password': password,
'tenant_name': tenant, 'auth_url': auth_url,
'ca_cert': path_to_cert, 'insecure': insecure,
'endpoint_url': make_endpoint(neutron_endpoint)}
self.neutron = neutronclient.Client(**neutron_args)
@property
def keystone(self):
return KeystoneClient(session=self.keystone_session)
nova_endpoint = self.keystone.service_catalog.url_for(
service_type='compute', endpoint_type='publicURL')
nova_args = {'username': user, 'api_key': password,
'project_id': tenant, 'auth_url': auth_url,
'cacert': path_to_cert, 'insecure': insecure,
'bypass_url': make_endpoint(nova_endpoint),
'auth_token': token}
self.nova = NovaClient(**nova_args)
@property
def glance(self):
endpoint = self.__make_endpoint(
self._get_url_for_svc(service_type='image'))
return GlanceClient(
session=self.keystone_session,
endpoint=endpoint,
endpoint_override=endpoint)
cinder_endpoint = self.keystone.service_catalog.url_for(
service_type='volume', endpoint_type='publicURL')
cinder_args = {'version': 1, 'username': user,
'api_key': password, 'project_id': tenant,
'auth_url': auth_url, 'cacert': path_to_cert,
'insecure': insecure,
'bypass_url': make_endpoint(cinder_endpoint)}
self.cinder = cinderclient.Client(**cinder_args)
@property
def neutron(self):
endpoint = self.__make_endpoint(
self._get_url_for_svc(service_type='network'))
return NeutronClient(
session=self.keystone_session,
endpoint_override=endpoint)
glance_endpoint = self.keystone.service_catalog.url_for(
service_type='image', endpoint_type='publicURL')
LOGGER.debug('Glance endpoint is {0}'.format(
make_endpoint(glance_endpoint)))
glance_args = {'endpoint': make_endpoint(glance_endpoint),
'token': token,
'cacert': path_to_cert,
'insecure': insecure}
self.glance = GlanceClient(**glance_args)
@property
def nova(self):
endpoint = self.__make_endpoint(
self._get_url_for_svc(service_type='compute'))
return NovaClient(
version='2',
session=self.keystone_session,
endpoint_override=endpoint)
heat_endpoint = self.keystone.service_catalog.url_for(
service_type='orchestration', endpoint_type='publicURL')
@property
def cinder(self):
endpoint = self.__make_endpoint(
self._get_url_for_svc(service_type='volume'))
return CinderClient(
version='1',
session=self.keystone_session,
endpoint_override=endpoint)
heat_args = {'endpoint': make_endpoint(heat_endpoint),
'token': token,
'cacert': path_to_cert,
'insecure': insecure}
self.heat = HeatClient(**heat_args)
@property
def heat(self):
endpoint = self.__make_endpoint(
self._get_url_for_svc(service_type='orchestration'))
return HeatClient(
session=self.keystone_session,
endpoint_override=endpoint)
@property
def ironic(self):
try:
ironic_endpoint = self.keystone.service_catalog.url_for(
service_type='baremetal',
endpoint_type='publicURL')
self.ironic = ironicclient.get_client(
api_version=1,
os_auth_token=token,
ironic_url=make_endpoint(ironic_endpoint), insecure=True)
endpoint = self.__make_endpoint(
self._get_url_for_svc(service_type='baremetal'))
return IronicClient(
version='1',
session=self.keystone_session,
insecure=True,
endpoint_override=endpoint
)
except ClientException as e:
LOGGER.warning('Could not initialize ironic client {0}'.format(e))
logger.warning('Could not initialize ironic client {0}'.format(e))
raise
@property
def keystone_access(self):
return self.__keystone_auth.get_access(session=self.keystone_session)
def _get_url_for_svc(
self, service_type=None, interface='public',
region_name=None, service_name=None,
service_id=None, endpoint_id=None
):
return self.keystone_access.service_catalog.url_for(
service_type=service_type, interface=interface,
region_name=region_name, service_name=service_name,
service_id=service_id, endpoint_id=endpoint_id
)
def goodbye_security(self):
secgroup_list = self.nova.security_groups.list()
LOGGER.debug("Security list is {0}".format(secgroup_list))
logger.debug("Security list is {0}".format(secgroup_list))
secgroup_id = [i.id for i in secgroup_list if i.name == 'default'][0]
LOGGER.debug("Id of security group default is {0}".format(
logger.debug("Id of security group default is {0}".format(
secgroup_id))
LOGGER.debug('Permit all TCP and ICMP in security group default')
logger.debug('Permit all TCP and ICMP in security group default')
self.nova.security_group_rules.create(secgroup_id,
ip_protocol='tcp',
from_port=1,
@ -143,15 +178,16 @@ class Common(object):
return self.glance.images.delete(image_id)
def create_key(self, key_name):
LOGGER.debug('Try to create key {0}'.format(key_name))
logger.debug('Try to create key {0}'.format(key_name))
return self.nova.keypairs.create(key_name)
def create_instance(self, flavor_name='test_flavor', ram=64, vcpus=1,
disk=1, server_name='test_instance', image_name=None,
neutron_network=True, label=None):
LOGGER.debug('Try to create instance')
logger.debug('Try to create instance')
start_time = time.time()
exc_type, exc_value, exc_traceback = None, None, None
while time.time() - start_time < 100:
try:
if image_name:
@ -160,9 +196,13 @@ class Common(object):
else:
image = [i.id for i in self.nova.images.list()]
break
except:
pass
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
logger.warning('Ignoring exception: {!r}'.format(e))
logger.debug(traceback.format_exc())
else:
if all((exc_type, exc_traceback, exc_value)):
six.reraise(exc_type, exc_value, exc_traceback)
raise Exception('Can not get image')
kwargs = {}
@ -171,16 +211,16 @@ class Common(object):
network = self.nova.networks.find(label=net_label)
kwargs['nics'] = [{'net-id': network.id, 'v4-fixed-ip': ''}]
LOGGER.info('image uuid is {0}'.format(image))
logger.info('image uuid is {0}'.format(image))
flavor = self.nova.flavors.create(
name=flavor_name, ram=ram, vcpus=vcpus, disk=disk)
LOGGER.info('flavor is {0}'.format(flavor.name))
logger.info('flavor is {0}'.format(flavor.name))
server = self.nova.servers.create(
name=server_name, image=image[0], flavor=flavor, **kwargs)
LOGGER.info('server is {0}'.format(server.name))
logger.info('server is {0}'.format(server.name))
return server
@LOGWRAP
@logwrap
def get_instance_detail(self, server):
details = self.nova.servers.get(server)
return details
@ -193,13 +233,13 @@ class Common(object):
try:
_verify_instance_state()
except AssertionError:
LOGGER.debug('Instance is not {0}, lets provide it the last '
logger.debug('Instance is not {0}, lets provide it the last '
'chance and sleep 60 sec'.format(expected_state))
time.sleep(60)
_verify_instance_state()
def delete_instance(self, server):
LOGGER.debug('Try to delete instance')
logger.debug('Try to delete instance')
self.nova.servers.delete(server)
def create_flavor(self, name, ram, vcpus, disk, flavorid="auto",
@ -211,29 +251,28 @@ class Common(object):
def delete_flavor(self, flavor):
return self.nova.flavors.delete(flavor)
def _get_keystoneclient(self, username, password, tenant_name, auth_url,
retries=3, ca_cert=None, insecure=False):
keystone = None
for i in range(retries):
def __start_keystone_session(
self, retries=3, ca_cert=None, insecure=not VERIFY_SSL):
exc_type, exc_value, exc_traceback = None, None, None
for i in xrange(retries):
try:
if ca_cert:
keystone = KeystoneClient(username=username,
password=password,
tenant_name=tenant_name,
auth_url=auth_url,
cacert=ca_cert,
insecure=insecure)
if insecure:
self.keystone_session = KeystoneSession(
auth=self.__keystone_auth, verify=False)
elif ca_cert:
self.keystone_session = KeystoneSession(
auth=self.__keystone_auth, verify=ca_cert)
else:
keystone = KeystoneClient(username=username,
password=password,
tenant_name=tenant_name,
auth_url=auth_url)
break
except ClientException as e:
err = "Try nr {0}. Could not get keystone client, error: {1}"
LOGGER.warning(err.format(i + 1, e))
self.keystone_session = KeystoneSession(
auth=self.__keystone_auth)
self.keystone_session.get_auth_headers()
return
except ClientException as exc:
exc_type, exc_value, exc_traceback = sys.exc_info()
err = "Try nr {0}. Could not get keystone token, error: {1}"
logger.warning(err.format(i + 1, exc))
time.sleep(5)
if not keystone:
raise
return keystone
if exc_type and exc_traceback and exc_value:
six.reraise(exc_type, exc_value, exc_traceback)
raise RuntimeError()

View File

@ -14,10 +14,19 @@
import json
import traceback
import urllib2
from keystoneauth1 import exceptions
from keystoneauth1.identity import V2Password
from keystoneauth1.session import Session as KeystoneSession
from keystoneclient.v2_0 import Client as KeystoneClient
from keystoneclient import exceptions
# pylint: disable=import-error
# noinspection PyUnresolvedReferences
from six.moves.urllib import request
# noinspection PyUnresolvedReferences
from six.moves.urllib.error import HTTPError
# pylint: enable=import-error
import requests
from fuelweb_test import logger
@ -30,17 +39,21 @@ class HTTPClient(object):
self.keystone_url = keystone_url
self.creds = dict(credentials, **kwargs)
self.keystone = None
self.opener = urllib2.build_opener(urllib2.HTTPHandler)
self.session = None
self.opener = request.build_opener(request.HTTPHandler)
def authenticate(self):
try:
logger.info('Initialize keystoneclient with url %s',
self.keystone_url)
self.keystone = KeystoneClient(
auth_url=self.keystone_url, **self.creds)
# it depends on keystone version, some versions doing auth
# explicitly some don't, but we are making it explicitly always
self.keystone.authenticate()
auth = V2Password(
auth_url=self.keystone_url,
username=self.creds['username'],
password=self.creds['password'],
tenant_name=self.creds['tenant_name'])
# TODO: in v3 project_name
self.session = KeystoneSession(auth=auth, verify=False)
self.keystone = KeystoneClient(session=self.session)
logger.debug('Authorization token is successfully updated')
except exceptions.AuthorizationFailure:
logger.warning(
@ -51,7 +64,7 @@ class HTTPClient(object):
def token(self):
if self.keystone is not None:
try:
return self.keystone.auth_token
return self.session.get_token()
except exceptions.AuthorizationFailure:
logger.warning(
'Cant establish connection to keystone with url %s',
@ -60,37 +73,45 @@ class HTTPClient(object):
logger.warning("Keystone returned unauthorized error, trying "
"to pass authentication.")
self.authenticate()
return self.keystone.auth_token
return self.session.get_token()
return None
def get(self, endpoint):
req = urllib2.Request(self.url + endpoint)
req = request.Request(self.url + endpoint)
return self._open(req)
def post(self, endpoint, data=None, content_type="application/json"):
if not data:
data = {}
req = urllib2.Request(self.url + endpoint, data=json.dumps(data))
req = request.Request(self.url + endpoint, data=json.dumps(data))
req.add_header('Content-Type', content_type)
return self._open(req)
def put(self, endpoint, data=None, content_type="application/json"):
if not data:
data = {}
req = urllib2.Request(self.url + endpoint, data=json.dumps(data))
req = request.Request(self.url + endpoint, data=json.dumps(data))
req.add_header('Content-Type', content_type)
req.get_method = lambda: 'PUT'
return self._open(req)
def delete(self, endpoint):
req = urllib2.Request(self.url + endpoint)
req = request.Request(self.url + endpoint)
req.get_method = lambda: 'DELETE'
return self._open(req)
def _open(self, req):
try:
return self._get_response(req)
except urllib2.HTTPError as e:
except HTTPError as e:
if e.code == 308:
logger.info(e.read())
url = req.get_full_url()
req = requests.get(url, headers={'X-Auth-Token': self.token})
if req.status_code in [200]:
return req.json()
else:
req.raise_for_status()
if e.code == 401:
logger.warning('Authorization failure: {0}'.format(e.read()))
self.authenticate()
@ -121,10 +142,10 @@ class HTTPClientZabbix(object):
def __init__(self, url):
self.url = url
self.opener = urllib2.build_opener(urllib2.HTTPHandler)
self.opener = request.build_opener(request.HTTPHandler)
def get(self, endpoint=None, cookie=None):
req = urllib2.Request(self.url + endpoint)
req = request.Request(self.url + endpoint)
if cookie:
req.add_header('cookie', cookie)
return self.opener.open(req)
@ -133,7 +154,7 @@ class HTTPClientZabbix(object):
cookie=None):
if not data:
data = {}
req = urllib2.Request(self.url + endpoint, data=json.dumps(data))
req = request.Request(self.url + endpoint, data=json.dumps(data))
req.add_header('Content-Type', content_type)
if cookie:
req.add_header('cookie', cookie)

View File

@ -12,13 +12,14 @@
# License for the specific language governing permissions and limitations
# under the License.
import paramiko
from proboscis import asserts
import random
import time
from devops.error import TimeoutError
from devops.helpers import helpers
import paramiko
from proboscis import asserts
from fuelweb_test.helpers import common
from fuelweb_test import logger
@ -116,7 +117,7 @@ class OpenStackActions(common.Common):
" is {0}".format(self.get_instance_detail(srv).status))
def create_server_for_migration(self, neutron=True, scenario='',
timeout=100, file=None, key_name=None,
timeout=100, filename=None, key_name=None,
label=None, flavor=1, **kwargs):
name = "test-serv" + str(random.randint(1, 0x7fffffff))
security_group = {}
@ -125,12 +126,12 @@ class OpenStackActions(common.Common):
with open(scenario, "r+") as f:
scenario = f.read()
except Exception as exc:
logger.info("Error opening file: %s" % exc)
logger.info("Error opening file: {:s}".format(exc))
raise Exception()
image_id = self._get_cirros_image().id
security_group[self.keystone.tenant_id] =\
security_group[self.keystone_access.tenant_id] =\
self.create_sec_group_for_ssh()
security_group = [security_group[self.keystone.tenant_id].name]
security_groups = [security_group[self.keystone_access.tenant_id].name]
if neutron:
net_label = label if label else 'net04'
@ -138,15 +139,15 @@ class OpenStackActions(common.Common):
if net.label == net_label]
kwargs.update({'nics': [{'net-id': network[0]}],
'security_groups': security_group})
'security_groups': security_groups})
else:
kwargs.update({'security_groups': security_group})
kwargs.update({'security_groups': security_groups})
srv = self.nova.servers.create(name=name,
image=image_id,
flavor=flavor,
userdata=scenario,
files=file,
files=filename,
key_name=key_name,
**kwargs)
try:
@ -195,9 +196,8 @@ class OpenStackActions(common.Common):
flip = self.neutron.create_floatingip(body)
# Wait active state for port
port_id = flip['floatingip']['port_id']
helpers.wait(
lambda:
self.neutron.show_port(port_id)['port']['status'] == "ACTIVE")
helpers.wait(lambda: self.neutron.show_port(
port_id)['port']['status'] == "ACTIVE")
return flip['floatingip']
fl_ips_pool = self.nova.floating_ip_pools.list()
@ -275,8 +275,9 @@ class OpenStackActions(common.Common):
server = self.get_instance_detail(server.id)
return server
def create_volume(self, size=1, image_id=None):
volume = self.cinder.volumes.create(size=size, imageRef=image_id)
def create_volume(self, size=1, image_id=None, **kwargs):
volume = self.cinder.volumes.create(size=size, imageRef=image_id,
**kwargs)
helpers.wait(
lambda: self.cinder.volumes.get(volume.id).status == "available",
timeout=100)
@ -308,18 +309,19 @@ class OpenStackActions(common.Common):
def get_hosts_for_migr(self, srv_host_name):
# Determine which host is available for live migration
host_list = filter(lambda host: host.host_name != srv_host_name,
self.nova.hosts.list())
return filter(lambda host: host._info['service'] == 'compute',
host_list)
return [
host for host in self.nova.hosts.list()
if host.host_name != srv_host_name and
host._info['service'] == 'compute']
def get_md5sum(self, file_path, controller_ssh, vm_ip, creds=()):
logger.info("Get file md5sum and compare it with previous one")
out = self.execute_through_host(
controller_ssh, vm_ip, "md5sum %s" % file_path, creds)
controller_ssh, vm_ip, "md5sum {:s}".format(file_path), creds)
return out['stdout']
def execute_through_host(self, ssh, vm_host, cmd, creds=()):
@staticmethod
def execute_through_host(ssh, vm_host, cmd, creds=()):
logger.debug("Making intermediate transport")
intermediate_transport = ssh._ssh.get_transport()
@ -547,16 +549,17 @@ class OpenStackActions(common.Common):
def get_vip(self, vip):
return self.neutron.show_vip(vip)
def get_nova_instance_ip(self, srv, net_name='novanetwork', type='fixed'):
@staticmethod
def get_nova_instance_ip(srv, net_name='novanetwork', addrtype='fixed'):
for network_label, address_list in srv.addresses.items():
if network_label != net_name:
continue
for addr in address_list:
if addr['OS-EXT-IPS:type'] == type:
if addr['OS-EXT-IPS:type'] == addrtype:
return addr['addr']
raise Exception("Instance {0} doesn't have {1} address for network "
"{2}, available addresses: {3}".format(srv.id,
type,
addrtype,
net_name,
srv.addresses))

View File

@ -4,18 +4,23 @@ paramiko>=1.16.0 # LGPL
proboscis==1.2.6.0
ipaddr
junitxml>=0.7.0
netaddr
pyOpenSSL>=0.14
python-glanceclient>=0.18.0
python-keystoneclient>=0.3.2
python-novaclient>=2.15.0
python-cinderclient>=1.0.5
python-neutronclient>=2.6.0
python-ironicclient>=0.8.0
python-heatclient>=0.6.0
netaddr>=0.7.12,!=0.7.16 # BSD
pyOpenSSL>=0.14 # Apache-2.0
Sphinx # BSD # Not required for tests, but required to build docs (pbr)
docutils # Not required for tests, but required to build docs (pbr)
markupsafe # Not required for tests, but required to build docs (pbr)
pytz>=2013.6 # MIT # Not required for tests, but required to build docs (pbr)
keystoneauth1>=2.1.0 # Apache-2.0
python-glanceclient>=1.2.0 # Apache-2.0
python-keystoneclient>=1.6.0,!=1.8.0,!=2.1.0 # Apache-2.0
python-novaclient>=2.29.0,!=2.33.0 # Apache-2.0
python-cinderclient>=1.3.1 # Apache-2.0
python-neutronclient>=2.6.0,!=4.1.0 # Apache-2.0
python-ironicclient>=1.1.0 # Apache-2.0
python-heatclient>=0.6.0 # Apache-2.0
oslo.i18n>=3.1.0
six
Jinja2
six>=1.9.0 # MIT
Jinja2>=2.8 # BSD License (3 clause)
AllPairs==2.0.1
launchpadlib
beautifulsoup4>=4.2.0