346 lines
14 KiB
Python
346 lines
14 KiB
Python
# Copyright 2017 Red Hat
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
import functools
|
|
import json
|
|
import six
|
|
import subprocess
|
|
import time
|
|
|
|
from oslo_log import log as logging
|
|
from tempest import config
|
|
|
|
from novajoin_tempest_plugin.ipa import ipa_client
|
|
from novajoin_tempest_plugin.tests.scenario import manager
|
|
|
|
CONF = config.CONF
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
CONTAINER_HAPROXY_FILE = (
|
|
'/var/lib/config-data/puppet-generated/haproxy/etc/haproxy/haproxy.cfg'
|
|
)
|
|
HAPROXY_FILE = "/etc/haproxy/haproxy.cfg"
|
|
|
|
|
|
class NovajoinScenarioTest(manager.ScenarioTest):
|
|
|
|
credentials = ['primary', 'admin']
|
|
|
|
def setUp(self):
|
|
super(NovajoinScenarioTest, self).setUp()
|
|
|
|
@classmethod
|
|
def setup_credentials(cls):
|
|
cls.set_network_resources()
|
|
super(NovajoinScenarioTest, cls).setup_credentials()
|
|
|
|
@classmethod
|
|
def skip_checks(cls):
|
|
super(NovajoinScenarioTest, cls).skip_checks()
|
|
if not CONF.service_available.novajoin:
|
|
raise cls.skipException("Novajoin is not enabled")
|
|
|
|
@classmethod
|
|
def setup_clients(cls):
|
|
super(NovajoinScenarioTest, cls).setup_clients()
|
|
cls.ipa_client = ipa_client.IPAClient()
|
|
|
|
def retry_with_timeout(func):
|
|
@functools.wraps(func)
|
|
def wrapper_retry_with_timeout(*args, **kwargs):
|
|
start = int(time.time())
|
|
timeout = 300
|
|
result = func(*args, **kwargs)
|
|
while not result and (int(time.time()) - start < timeout):
|
|
time.sleep(30)
|
|
result = func(*args, **kwargs)
|
|
assert result
|
|
return wrapper_retry_with_timeout
|
|
|
|
@retry_with_timeout
|
|
def verify_host_registered_with_ipa(self, host, add_domain=True):
|
|
if add_domain:
|
|
host = self.add_domain_to_host(host)
|
|
result = self.ipa_client.find_host(host)
|
|
return result['count'] > 0
|
|
|
|
@retry_with_timeout
|
|
def verify_host_not_registered_with_ipa(self, host, add_domain=True):
|
|
if add_domain:
|
|
host = self.add_domain_to_host(host)
|
|
result = self.ipa_client.find_host(host)
|
|
return result['count'] == 0
|
|
|
|
def add_domain_to_host(self, host):
|
|
host = '{host}.{domain}'.format(
|
|
host=host,
|
|
domain=self.ipa_client.domain)
|
|
return host
|
|
|
|
@retry_with_timeout
|
|
def verify_host_has_keytab(self, host, add_domain=True):
|
|
if add_domain:
|
|
host = self.add_domain_to_host(host)
|
|
result = self.ipa_client.show_host(host)['result']
|
|
return result['has_keytab']
|
|
|
|
@retry_with_timeout
|
|
def verify_service_created(self, service, host):
|
|
service_principal = self.get_service_principal(host, service)
|
|
result = self.ipa_client.find_service(service_principal)
|
|
return result['count'] > 0
|
|
|
|
@retry_with_timeout
|
|
def verify_service_managed_by_host(self, service, host):
|
|
service_principal = self.get_service_principal(host, service)
|
|
return self.ipa_client.service_managed_by_host(service_principal,
|
|
host)
|
|
|
|
@retry_with_timeout
|
|
def verify_service_deleted(self, service, host):
|
|
service_principal = self.get_service_principal(host, service)
|
|
result = self.ipa_client.find_service(service_principal)
|
|
return result['count'] == 0
|
|
|
|
def verify_compact_services_deleted(self, services, host):
|
|
for (service, networks) in services.items():
|
|
for network in networks:
|
|
subhost = '{host}.{network}.{domain}'.format(
|
|
host=host, network=network, domain=self.ipa_client.domain
|
|
)
|
|
self.verify_service_deleted(service, subhost)
|
|
|
|
def verify_managed_services_deleted(self, services):
|
|
for principal in services:
|
|
service = principal.split('/', 1)[0]
|
|
host = principal.split('/', 1)[1]
|
|
self.verify_service_deleted(service, host)
|
|
|
|
def get_service_cert(self, service, host):
|
|
service_principal = self.get_service_principal(host, service)
|
|
return self.ipa_client.get_service_cert(service_principal)
|
|
|
|
def get_service_principal(self, host, service):
|
|
return '{service}/{hostname}@{realm}'.format(
|
|
service=service, hostname=host, realm=self.ipa_client.realm
|
|
)
|
|
|
|
def verify_host_is_ipaclient(self, hostip, user, keypair):
|
|
cmd = "id admin"
|
|
private_key = keypair['private_key']
|
|
ssh_client = self.get_remote_client(hostip, user, private_key)
|
|
result = ssh_client.exec_command(cmd)
|
|
params = ['uid', 'gid', 'groups']
|
|
self.assertTrue(all(x in result for x in params))
|
|
|
|
def verify_overcloud_host_is_ipaclient(self, hostip, user):
|
|
cmd = 'id admin'
|
|
result = self.execute_on_controller(user, hostip, cmd)
|
|
params = ['uid', 'gid', 'groups']
|
|
self.assertTrue(all(x in result for x in params))
|
|
|
|
def verify_cert_tracked(self, hostip, user, keypair, cert_id):
|
|
cmd = 'sudo getcert list -i {certid}'.format(certid=cert_id)
|
|
private_key = keypair['private_key']
|
|
ssh_client = self.get_remote_client(hostip, user, private_key)
|
|
result = ssh_client.exec_command(cmd)
|
|
self.assertTrue('track: yes' in result)
|
|
|
|
def verify_overcloud_cert_tracked(self, hostip, user, cert_id):
|
|
cmd = 'sudo getcert list -i {certid}'.format(certid=cert_id)
|
|
result = self.execute_on_controller(user, hostip, cmd)
|
|
self.assertTrue('track: yes' in result)
|
|
|
|
@retry_with_timeout
|
|
def verify_cert_revoked(self, serial):
|
|
# verify that the given certificate has been revoked
|
|
result = self.ipa_client.show_cert(serial)['result']
|
|
return result['revoked']
|
|
|
|
def get_compact_services(self, metadata):
|
|
# compact key-per-service
|
|
compact_services = {key.split('_', 2)[-1]: json.loads(value)
|
|
for key, value in six.iteritems(metadata)
|
|
if key.startswith('compact_service_')}
|
|
if compact_services:
|
|
return compact_services
|
|
# legacy compact json format
|
|
if 'compact_services' in metadata:
|
|
return json.loads(metadata['compact_services'])
|
|
|
|
return None
|
|
|
|
def verify_compact_services(self, services, host,
|
|
host_ip, verify_certs=False):
|
|
for (service, networks) in services.items():
|
|
for network in networks:
|
|
subhost = '{host}.{network}.{domain}'.format(
|
|
host=host, network=network, domain=self.ipa_client.domain
|
|
)
|
|
LOG.debug("SUBHOST: %s", subhost)
|
|
self.verify_service(service, subhost, host_ip,
|
|
verify_certs, network)
|
|
|
|
def verify_service(self, service, host, host_ip,
|
|
verify_certs=False, network=False):
|
|
LOG.debug("verifying: %s %s", service, host)
|
|
if network:
|
|
LOG.debug("verifying network %s", network)
|
|
self.verify_host_registered_with_ipa(host, add_domain=False)
|
|
self.verify_service_created(service, host)
|
|
self.verify_service_managed_by_host(service, host)
|
|
if verify_certs:
|
|
self.verify_service_cert(service, host, host_ip, network)
|
|
LOG.debug("verified: %s %s ", service, host)
|
|
|
|
def verify_service_cert(self, service, host, host_ip, network=None):
|
|
LOG.debug("Verifying cert for %s %s", service, host)
|
|
|
|
if not self.network_defined(host, network, host_ip):
|
|
# if the network is not enabled for this host
|
|
# no cert will be requested
|
|
LOG.debug("No network defined for {network} on {host}.".format(
|
|
network=network, host=host))
|
|
return
|
|
serial = self.get_service_cert(service, host)
|
|
|
|
internal_controllers = ['{controller}.internalapi.{domain}'.format(
|
|
controller=ctl, domain=self.ipa_client.domain) for ctl in
|
|
CONF.novajoin.tripleo_controllers]
|
|
|
|
# TODO(alee) Need to understand why mysql is different
|
|
if service == 'mysql' and host in internal_controllers:
|
|
pass
|
|
else:
|
|
if serial is None:
|
|
LOG.error("Cert NOT verified for %s %s", service, host)
|
|
self.assertTrue(serial is not None)
|
|
LOG.debug("Cert verified for %s %s", service, host)
|
|
|
|
def network_defined(self, host, network, host_ip):
|
|
"""Confirm network is defined on host."""
|
|
if network == 'internalapi':
|
|
network = 'internal_api'
|
|
if network == 'storagemgmt':
|
|
network = 'storage_mgmt'
|
|
cmd = ('sudo hiera -c /etc/puppet/hiera.yaml fqdn_{network}'.format(
|
|
network=network))
|
|
result = self.execute_on_controller('heat-admin', host_ip, cmd)
|
|
return result.strip() != 'nil'
|
|
|
|
def verify_managed_services(self, services, verify_certs=False):
|
|
for principal in services:
|
|
service = principal.split('/', 1)[0]
|
|
host = principal.split('/', 1)[1]
|
|
self.verify_service(service, host, verify_certs)
|
|
|
|
def verify_overcloud_tls_connection(self, controller_ip, user, hostport):
|
|
"""Check TLS connection. Failure will raise an exception"""
|
|
cmd = ('echo \'GET / HTTP/1.0\r\n\' | openssl s_client '
|
|
'-connect {hostport} -tls1_2'.format(hostport=hostport))
|
|
self.execute_on_controller(user, controller_ip, cmd)
|
|
|
|
def get_pcs_node(self, vip, controller_ip, user, hostport):
|
|
"""Get controller node that hosts vip"""
|
|
cmd = ('sudo pcs status |grep {vip}| '
|
|
'sed \'s/.*Started \(.*\)/\\1/\''.format(vip=vip))
|
|
return self.execute_on_controller(user, controller_ip, cmd).strip()
|
|
|
|
def get_server_id(self, name):
|
|
params = {'all_tenants': '', 'name': name}
|
|
resp = self.servers_client.list_servers(detail=True, **params)
|
|
servers = resp['servers']
|
|
if servers:
|
|
links = resp['servers'][0]['links']
|
|
for link in links:
|
|
if link['rel'] == 'self':
|
|
href = link['href']
|
|
return href.split('/')[-1]
|
|
return None
|
|
|
|
def get_overcloud_server_ip(self, host):
|
|
host_id = self.get_server_id(host)
|
|
host_data = self.servers_client.show_server(host_id)['server']
|
|
return self.get_server_ip(host_data)
|
|
|
|
def get_haproxy_cfg(self, user, controller_ip):
|
|
try:
|
|
# check containerized location first
|
|
cmd = 'sudo cat {fname}'.format(fname=CONTAINER_HAPROXY_FILE)
|
|
return self.execute_on_controller(user, controller_ip, cmd)
|
|
except subprocess.CalledProcessError:
|
|
# try non-containerized location
|
|
cmd = 'sudo cat {fname}'.format(fname=HAPROXY_FILE)
|
|
return self.execute_on_controller(user, controller_ip, cmd)
|
|
|
|
def get_rabbitmq_host(self, user, controller_ip):
|
|
return self.get_hiera(user, controller_ip, 'rabbitmq::ssl_interface')
|
|
|
|
def get_rabbitmq_port(self, user, controller_ip):
|
|
return self.get_hiera(user, controller_ip, 'rabbitmq::ssl_port')
|
|
|
|
def get_libvirt_port(self, user, compute_ip):
|
|
# TODO(alee) Get from hiera nova::migration::libvirt::listen_address
|
|
return "16514"
|
|
|
|
def get_hiera(self, user, host_ip, parameter):
|
|
cmd = ('sudo hiera -c /etc/puppet/hiera.yaml '
|
|
'{parameter}'.format(parameter=parameter))
|
|
return self.execute_on_controller(user, host_ip, cmd).rstrip()
|
|
|
|
def verify_mysql_tls_connection(self, user, host_ip):
|
|
cmd = (CONF.novajoin.mysql_command +
|
|
" --ssl -e \"SHOW SESSION STATUS LIKE 'Ssl_version';\"")
|
|
result = self.execute_on_controller(user, host_ip, cmd)
|
|
self.assertTrue('TLS' in result)
|
|
|
|
def verify_mysql_access_with_ssl(self,
|
|
user,
|
|
host_ip,
|
|
dbuser,
|
|
dbhost,
|
|
dbpassword):
|
|
sql = "SHOW SESSION STATUS LIKE \'Ssl_version\';"
|
|
cmd = (CONF.novajoin.mysql_command +
|
|
' --ssl -u {user} -h {host} --password={password} '
|
|
'-e \"{sql}\"'.format(user=dbuser,
|
|
host=dbhost,
|
|
password=dbpassword,
|
|
sql=sql))
|
|
result = self.execute_on_controller(user, host_ip, cmd)
|
|
self.assertTrue('TLS' in result)
|
|
|
|
def verify_mysql_access_without_ssl(self,
|
|
user,
|
|
host_ip,
|
|
dbuser,
|
|
dbhost,
|
|
dbpassword):
|
|
cmd = (CONF.novajoin.mysql_command +
|
|
' -u {user} -h {host} --password={password} '
|
|
'-e \"SHOW DATABASES;\"'.format(user=dbuser,
|
|
host=dbhost,
|
|
password=dbpassword))
|
|
self.assertRaises(subprocess.CalledProcessError,
|
|
self.execute_on_controller,
|
|
user, host_ip, cmd)
|
|
|
|
def execute_on_controller(self, user, hostip, target_cmd):
|
|
keypair = '/home/stack/.ssh/id_rsa'
|
|
cmd = ['ssh', '-i', keypair,
|
|
'{user}@{hostip}'.format(user=user, hostip=hostip),
|
|
'-C', target_cmd]
|
|
stdout = subprocess.check_output(cmd)
|
|
return stdout.decode('UTF-8')
|