# Copyright 2017 Red Hat # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import functools import json import six import subprocess import time from oslo_log import log as logging from tempest import config from novajoin_tempest_plugin.ipa import ipa_client from novajoin_tempest_plugin.tests.scenario import manager CONF = config.CONF LOG = logging.getLogger(__name__) CONTAINER_HAPROXY_FILE = ( '/var/lib/config-data/puppet-generated/haproxy/etc/haproxy/haproxy.cfg' ) HAPROXY_FILE = "/etc/haproxy/haproxy.cfg" class NovajoinScenarioTest(manager.ScenarioTest): credentials = ['primary', 'admin'] def setUp(self): super(NovajoinScenarioTest, self).setUp() @classmethod def setup_credentials(cls): cls.set_network_resources() super(NovajoinScenarioTest, cls).setup_credentials() @classmethod def skip_checks(cls): super(NovajoinScenarioTest, cls).skip_checks() if not CONF.service_available.novajoin: raise cls.skipException("Novajoin is not enabled") @classmethod def setup_clients(cls): super(NovajoinScenarioTest, cls).setup_clients() cls.ipa_client = ipa_client.IPAClient() def retry_with_timeout(func): @functools.wraps(func) def wrapper_retry_with_timeout(*args, **kwargs): start = int(time.time()) timeout = 300 result = func(*args, **kwargs) while not result and (int(time.time()) - start < timeout): time.sleep(30) result = func(*args, **kwargs) assert result return wrapper_retry_with_timeout @retry_with_timeout def verify_host_registered_with_ipa(self, host, add_domain=True): if add_domain: host = self.add_domain_to_host(host) result = self.ipa_client.find_host(host) return result['count'] > 0 @retry_with_timeout def verify_host_not_registered_with_ipa(self, host, add_domain=True): if add_domain: host = self.add_domain_to_host(host) result = self.ipa_client.find_host(host) return result['count'] == 0 def add_domain_to_host(self, host): host = '{host}.{domain}'.format( host=host, domain=self.ipa_client.domain) return host @retry_with_timeout def verify_host_has_keytab(self, host, add_domain=True): if add_domain: host = self.add_domain_to_host(host) result = self.ipa_client.show_host(host)['result'] return result['has_keytab'] @retry_with_timeout def verify_service_created(self, service, host): service_principal = self.get_service_principal(host, service) result = self.ipa_client.find_service(service_principal) return result['count'] > 0 @retry_with_timeout def verify_service_managed_by_host(self, service, host): service_principal = self.get_service_principal(host, service) return self.ipa_client.service_managed_by_host(service_principal, host) @retry_with_timeout def verify_service_deleted(self, service, host): service_principal = self.get_service_principal(host, service) result = self.ipa_client.find_service(service_principal) return result['count'] == 0 def verify_compact_services_deleted(self, services, host): for (service, networks) in services.items(): for network in networks: subhost = '{host}.{network}.{domain}'.format( host=host, network=network, domain=self.ipa_client.domain ) self.verify_service_deleted(service, subhost) def verify_managed_services_deleted(self, services): for principal in services: service = principal.split('/', 1)[0] host = principal.split('/', 1)[1] self.verify_service_deleted(service, host) def get_service_cert(self, service, host): service_principal = self.get_service_principal(host, service) return self.ipa_client.get_service_cert(service_principal) def get_service_principal(self, host, service): return '{service}/{hostname}@{realm}'.format( service=service, hostname=host, realm=self.ipa_client.realm ) def verify_host_is_ipaclient(self, hostip, user, keypair): cmd = "id admin" private_key = keypair['private_key'] ssh_client = self.get_remote_client(hostip, user, private_key) result = ssh_client.exec_command(cmd) params = ['uid', 'gid', 'groups'] self.assertTrue(all(x in result for x in params)) def verify_overcloud_host_is_ipaclient(self, hostip, user): cmd = 'id admin' result = self.execute_on_controller(user, hostip, cmd) params = ['uid', 'gid', 'groups'] self.assertTrue(all(x in result for x in params)) def verify_cert_tracked(self, hostip, user, keypair, cert_id): cmd = 'sudo getcert list -i {certid}'.format(certid=cert_id) private_key = keypair['private_key'] ssh_client = self.get_remote_client(hostip, user, private_key) result = ssh_client.exec_command(cmd) self.assertTrue('track: yes' in result) def verify_overcloud_cert_tracked(self, hostip, user, cert_id): cmd = 'sudo getcert list -i {certid}'.format(certid=cert_id) result = self.execute_on_controller(user, hostip, cmd) self.assertTrue('track: yes' in result) @retry_with_timeout def verify_cert_revoked(self, serial): # verify that the given certificate has been revoked result = self.ipa_client.show_cert(serial)['result'] return result['revoked'] def get_compact_services(self, metadata): # compact key-per-service compact_services = {key.split('_', 2)[-1]: json.loads(value) for key, value in six.iteritems(metadata) if key.startswith('compact_service_')} if compact_services: return compact_services # legacy compact json format if 'compact_services' in metadata: return json.loads(metadata['compact_services']) return None def verify_compact_services(self, services, host, host_ip, verify_certs=False): for (service, networks) in services.items(): for network in networks: subhost = '{host}.{network}.{domain}'.format( host=host, network=network, domain=self.ipa_client.domain ) LOG.debug("SUBHOST: %s", subhost) self.verify_service(service, subhost, host_ip, verify_certs, network) def verify_service(self, service, host, host_ip, verify_certs=False, network=False): LOG.debug("verifying: %s %s", service, host) if network: LOG.debug("verifying network %s", network) self.verify_host_registered_with_ipa(host, add_domain=False) self.verify_service_created(service, host) self.verify_service_managed_by_host(service, host) if verify_certs: self.verify_service_cert(service, host, host_ip, network) LOG.debug("verified: %s %s ", service, host) def verify_service_cert(self, service, host, host_ip, network=None): LOG.debug("Verifying cert for %s %s", service, host) if not self.network_defined(host, network, host_ip): # if the network is not enabled for this host # no cert will be requested LOG.debug("No network defined for {network} on {host}.".format( network=network, host=host)) return serial = self.get_service_cert(service, host) internal_controllers = ['{controller}.internalapi.{domain}'.format( controller=ctl, domain=self.ipa_client.domain) for ctl in CONF.novajoin.tripleo_controllers] # TODO(alee) Need to understand why mysql is different if service == 'mysql' and host in internal_controllers: pass else: if serial is None: LOG.error("Cert NOT verified for %s %s", service, host) self.assertTrue(serial is not None) LOG.debug("Cert verified for %s %s", service, host) def network_defined(self, host, network, host_ip): """Confirm network is defined on host.""" if network == 'internalapi': network = 'internal_api' if network == 'storagemgmt': network = 'storage_mgmt' cmd = ('sudo hiera -c /etc/puppet/hiera.yaml fqdn_{network}'.format( network=network)) result = self.execute_on_controller('heat-admin', host_ip, cmd) return result.strip() != 'nil' def verify_managed_services(self, services, verify_certs=False): for principal in services: service = principal.split('/', 1)[0] host = principal.split('/', 1)[1] self.verify_service(service, host, verify_certs) def verify_overcloud_tls_connection(self, controller_ip, user, hostport): """Check TLS connection. Failure will raise an exception""" cmd = ('echo \'GET / HTTP/1.0\r\n\' | openssl s_client ' '-connect {hostport} -tls1_2'.format(hostport=hostport)) self.execute_on_controller(user, controller_ip, cmd) def get_pcs_node(self, vip, controller_ip, user, hostport): """Get controller node that hosts vip""" cmd = ('sudo pcs status |grep {vip}| ' 'sed \'s/.*Started \(.*\)/\\1/\''.format(vip=vip)) return self.execute_on_controller(user, controller_ip, cmd).strip() def get_server_id(self, name): params = {'all_tenants': '', 'name': name} resp = self.servers_client.list_servers(detail=True, **params) print(resp) links = resp['servers'][0]['links'] for link in links: if link['rel'] == 'self': href = link['href'] return href.split('/')[-1] return None def get_overcloud_server_ip(self, host): host_id = self.get_server_id(host) host_data = self.servers_client.show_server(host_id)['server'] return self.get_server_ip(host_data) def get_haproxy_cfg(self, user, controller_ip): try: # check containerized location first cmd = 'sudo cat {fname}'.format(fname=CONTAINER_HAPROXY_FILE) return self.execute_on_controller(user, controller_ip, cmd) except subprocess.CalledProcessError: # try non-containerized location cmd = 'sudo cat {fname}'.format(fname=HAPROXY_FILE) return self.execute_on_controller(user, controller_ip, cmd) def get_rabbitmq_host(self, user, controller_ip): return self.get_hiera(user, controller_ip, 'rabbitmq::ssl_interface') def get_rabbitmq_port(self, user, controller_ip): return self.get_hiera(user, controller_ip, 'rabbitmq::ssl_port') def get_libvirt_port(self, user, compute_ip): # TODO(alee) Get from hiera nova::migration::libvirt::listen_address return "16514" def get_hiera(self, user, host_ip, parameter): cmd = ('sudo hiera -c /etc/puppet/hiera.yaml ' '{parameter}'.format(parameter=parameter)) return self.execute_on_controller(user, host_ip, cmd).rstrip() def verify_mysql_tls_connection(self, user, host_ip): cmd = "sudo mysql --ssl -e \"SHOW SESSION STATUS LIKE 'Ssl_version';\"" result = self.execute_on_controller(user, host_ip, cmd) self.assertTrue('TLS' in result) def verify_mysql_access_with_ssl(self, user, host_ip, dbuser, dbhost, dbpassword): sql = "SHOW SESSION STATUS LIKE \'Ssl_version\';" cmd = ('sudo mysql --ssl -u {user} -h {host} --password={password} ' '-e \"{sql}\"'.format(user=dbuser, host=dbhost, password=dbpassword, sql=sql)) result = self.execute_on_controller(user, host_ip, cmd) self.assertTrue('TLS' in result) def verify_mysql_access_without_ssl(self, user, host_ip, dbuser, dbhost, dbpassword): cmd = ('sudo mysql -u {user} -h {host} --password={password} ' '-e \"SHOW DATABASES;\"'.format(user=dbuser, host=dbhost, password=dbpassword)) self.assertRaises(subprocess.CalledProcessError, self.execute_on_controller, user, host_ip, cmd) def execute_on_controller(self, user, hostip, target_cmd): keypair = '/home/stack/.ssh/id_rsa' cmd = ['ssh', '-i', keypair, '{user}@{hostip}'.format(user=user, hostip=hostip), '-C', target_cmd] stdout = subprocess.check_output(cmd) return stdout.decode('UTF-8')