diff --git a/fuelweb_test/helpers/ssh_manager.py b/fuelweb_test/helpers/ssh_manager.py index 4625d11f9..07398d350 100644 --- a/fuelweb_test/helpers/ssh_manager.py +++ b/fuelweb_test/helpers/ssh_manager.py @@ -22,6 +22,7 @@ from devops.helpers.helpers import wait from devops.models.node import SSHClient from paramiko import RSAKey import six +import yaml from fuelweb_test import logger from fuelweb_test.helpers.metaclasses import SingletonMeta @@ -147,21 +148,26 @@ class SSHManager(object): def execute_on_remote(self, ip, cmd, port=22, err_msg=None, jsonify=False, assert_ec_equal=None, - raise_on_assert=True): + raise_on_assert=True, yamlify=False): """Execute ``cmd`` on ``remote`` and return result. :param ip: ip of host :param port: ssh port :param cmd: command to execute on remote host :param err_msg: custom error message + :param jsonify: bool, conflicts with yamlify :param assert_ec_equal: list of expected exit_code :param raise_on_assert: Boolean + :param yamlify: bool, conflicts with jsonify :return: dict :raise: Exception """ if assert_ec_equal is None: assert_ec_equal = [0] + if yamlify and jsonify: + raise ValueError('Conflicting arguments: yamlify and jsonify!') + result = self.execute(ip=ip, port=port, cmd=cmd) result['stdout_str'] = ''.join(result['stdout']).strip() @@ -201,15 +207,11 @@ class SSHManager(object): logger.debug(details_log) if jsonify: - try: - result['stdout_json'] = \ - self._json_deserialize(result['stdout_str']) - except Exception: - error_msg = ( - "Unable to deserialize output of command" - " '{0}' on host {1}".format(cmd, ip)) - logger.error(error_msg) - raise Exception(error_msg) + result['stdout_json'] = \ + self._json_deserialize(result['stdout_str']) + elif yamlify: + result['stdout_yaml'] = \ + self._yaml_deserialize(result['stdout_str']) return result @@ -226,15 +228,35 @@ class SSHManager(object): :raise: Exception """ if isinstance(json_string, list): - json_string = ''.join(json_string) + json_string = ''.join(json_string).strip() try: obj = json.loads(json_string) except Exception: - log_msg = "Unable to deserialize" - logger.error("{0}. Actual string:\n{1}".format(log_msg, - json_string)) - raise Exception(log_msg) + logger.error( + "Unable to deserialize. Actual string:\n" + "{}".format(json_string)) + raise + return obj + + @staticmethod + def _yaml_deserialize(yaml_string): + """ Deserialize yaml_string and return object + + :param yaml_string: string or list with yaml + :return: obj + :raise: Exception + """ + if isinstance(yaml_string, list): + yaml_string = ''.join(yaml_string).strip() + + try: + obj = yaml.safe_load(yaml_string) + except Exception: + logger.error( + "Unable to deserialize. Actual string:\n" + "{}".format(yaml_string)) + raise return obj def open_on_remote(self, ip, path, mode='r', port=22): diff --git a/fuelweb_test/tests/test_cli.py b/fuelweb_test/tests/test_cli.py index f0d6fe7db..6942def41 100644 --- a/fuelweb_test/tests/test_cli.py +++ b/fuelweb_test/tests/test_cli.py @@ -106,21 +106,28 @@ class CommandLineTest(test_cli_base.CommandLine): 1. Create environment using fuel-qa 2. Create a cluster using Fuel CLI 3. Add floating ranges for public network - 4. Provision a controller node using Fuel CLI - 5. Provision two compute+cinder nodes using Fuel CLI - 6. Deploy the controller node using Fuel CLI - 7. Deploy the compute+cinder nodes using Fuel CLI - 8. Verify network - 9. Compare floating ranges - 10. Check that all services work by 'https' - 11. Check that all services have domain name - 12. Find 'CN' value at the output: + 4. Get cluster settings + 5. Provision a controller node using Fuel CLI + 6. Provision two compute+cinder nodes using Fuel CLI + 7. Deploy the controller node using Fuel CLI + 8. Compare network settings after controller deployment task + 9. Deploy the compute+cinder nodes using Fuel CLI + 10. Compare network settings after compute deployment task + 11. Verify network + 12. Check that all services work by 'https' + 13. Check that all services have domain name + 14. Find 'CN' value at the output: CN value is equal to the value specified at certificate provided via Fuel UI - 13. Find keypair data at the output: + 15. Find keypair data at the output: Keypair data is equal to the value specified at certificate provided via Fuel UI - 14. Run OSTF + 16. Compare floating ranges + 17. Get deployment-info + 18. Get cluster settings after deployment task + 19. Compare cluster settings after deploy and before deploy + 20. Run OSTF + Duration 50m """ @@ -131,16 +138,19 @@ class CommandLineTest(test_cli_base.CommandLine): release_name=OPENSTACK_RELEASE)[0] admin_ip = self.ssh_manager.admin_ip # Create an environment + self.show_step(1) if NEUTRON_SEGMENT_TYPE: nst = '--nst={0}'.format(NEUTRON_SEGMENT_TYPE) else: nst = '' + self.show_step(2) cmd = ('fuel env create --name={0} --release={1} {2} --json'.format( self.__class__.__name__, release_id, nst)) env_result =\ self.ssh_manager.execute_on_remote(admin_ip, cmd, jsonify=True)['stdout_json'] cluster_id = env_result['id'] + self.show_step(3) # Update network parameters self.update_cli_network_configuration(cluster_id) # Change floating ranges @@ -160,6 +170,10 @@ class CommandLineTest(test_cli_base.CommandLine): # Update SSL configuration self.update_ssl_configuration(cluster_id) + # Get cluster settings before deploy + self.show_step(4) + cluster_settings = self.download_settings(cluster_id) + self.show_step(5) # Add and provision a controller node logger.info("Add to the cluster \ and start provisioning a controller node [{0}]".format(node_ids[0])) @@ -173,7 +187,7 @@ class CommandLineTest(test_cli_base.CommandLine): cmd, jsonify=True)['stdout_json'] self.assert_cli_task_success(task, timeout=30 * 60) - + self.show_step(6) # Add and provision 2 compute+cinder logger.info("Add to the cluster and start provisioning two " "compute+cinder nodes [{0},{1}]".format(node_ids[1], @@ -189,25 +203,42 @@ class CommandLineTest(test_cli_base.CommandLine): cmd, jsonify=True)['stdout_json'] self.assert_cli_task_success(task, timeout=10 * 60) + self.show_step(7) # Deploy the controller node cmd = ('fuel --env-id={0} node --deploy --node {1} --json'.format( cluster_id, node_ids[0])) task = self.ssh_manager.execute_on_remote(admin_ip, cmd, jsonify=True)['stdout_json'] + network_settings = self.get_networks(cluster_id) self.assert_cli_task_success(task, timeout=60 * 60) self.assert_all_tasks_completed(cluster_id=cluster_id) + self.show_step(8) + network_configuration = self.get_net_config_cli(task['id']) + assert_equal(network_settings, + network_configuration, + message='Network settings are not equal before' + ' and after deploy') + self.show_step(9) # Deploy the compute nodes cmd = ('fuel --env-id={0} node --deploy --node {1},{2} --json'.format( cluster_id, node_ids[1], node_ids[2])) task = self.ssh_manager.execute_on_remote(admin_ip, cmd, jsonify=True)['stdout_json'] + network_settings = self.get_networks(cluster_id) self.assert_cli_task_success(task, timeout=30 * 60) self.assert_all_tasks_completed(cluster_id=cluster_id) # Verify networks + self.show_step(10) + network_configuration = self.get_net_config_cli(task['id']) + assert_equal(network_settings, + network_configuration, + message='Network settings are not equal before' + ' and after deploy') + self.show_step(11) self.fuel_web.verify_network(cluster_id) controller_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles( cluster_id, ['controller']) @@ -217,6 +248,8 @@ class CommandLineTest(test_cli_base.CommandLine): endpoint_list = self.get_endpoints(controller_node) logger.info(endpoint_list) # Check protocol and domain names for endpoints + self.show_step(12) + self.show_step(13) for endpoint in endpoint_list: logger.debug(("Endpoint {0} use protocol {1}\ and have domain name {2}".format(endpoint['service_name'], @@ -228,12 +261,12 @@ class CommandLineTest(test_cli_base.CommandLine): assert_equal(endpoint['domain'], SSL_CN, message=( "{0} domain name not equal {1}.".format( endpoint['service_name'], SSL_CN))) - # Check SSL CNs + self.show_step(14) current_ssl_cn = self.get_current_ssl_cn(controller_node) logger.info(("CN before cluster deploy {0} \ and after deploy {1}".format(SSL_CN, current_ssl_cn))) assert_equal(SSL_CN, current_ssl_cn, message="SSL CNs are not equal") - # Check SSL certificate keypairs + self.show_step(15) with open(PATH_TO_PEM) as pem_file: old_ssl_keypair = pem_file.read().strip() current_ssl_keypair = self.get_current_ssl_keypair(controller_node) @@ -243,12 +276,27 @@ class CommandLineTest(test_cli_base.CommandLine): )) assert_equal(old_ssl_keypair, current_ssl_keypair, message="SSL keypairs are not equal") - # Check floating ranges are equal after cluster deploy + self.show_step(16) actual_floating_ranges = self.hiera_floating_ranges(controller_node) logger.info("Current floating ranges: {0}".format( actual_floating_ranges)) assert_equal(actual_floating_ranges, new_floating_range, message="Floating ranges are not equal") + # Get deployment task id + task_id = self.get_first_task_id_by_name(cluster_id, 'deployment') + self.show_step(17) + # Get deployment info + self.get_deployment_info_cli(task_id) + self.show_step(18) + # Get cluster settings after deploy + cluster_config = self.get_cluster_config_cli(task_id) + self.show_step(19) + # Compare cluster settings + assert_equal(cluster_settings, + cluster_config, + message='Cluster settings are not equal before' + ' and after deploy') + self.show_step(20) # Run OSTF self.fuel_web.run_ostf(cluster_id=cluster_id, test_sets=['ha', 'smoke', 'sanity']) @@ -377,21 +425,31 @@ class CommandLineTest(test_cli_base.CommandLine): 2. Create a cluster 3. Set replication factor 2 4. Set ceph usage for images, cinder for volumes - 5. Provision a controller node using Fuel CLI - 6. Provision one compute node using Fuel CLI - 7. Provision one cinder node using Fuel CLI - 8. Provision one mongo node using Fuel CLI - 9. Provision two ceph-osd nodes using Fuel CLI - 10. Provision one base-os node using Fuel CLI - 11. Leave 2 nodes in discover state - 12. Deploy the controller node using Fuel CLI - 13. Deploy the compute node using Fuel CLI - 14. Deploy the cinder node using Fuel CLI - 15. Deploy the mongo node using Fuel CLI - 16. Deploy the ceph-osd nodes using Fuel CLI - 17. Deploy the base-os node using Fuel CLI - 18. Check that nodes in discover state stay in it - 19. Run OSTF + 5. Get cluster settings before deploy + 6. Provision a controller node using Fuel CLI + 7. Provision one compute node using Fuel CLI + 8. Provision one cinder node using Fuel CLI + 9. Provision one mongo node using Fuel CLI + 10. Provision two ceph-osd nodes using Fuel CLI + 11. Provision one base-os node using Fuel CLI + 12. Leave 2 nodes in discover state + 13. Deploy the controller node using Fuel CLI + 14. Compare network settings after controller deployment task + 15. Deploy the compute node using Fuel CLI + 16. Compare network settings after compute deployment task + 17. Deploy the cinder node using Fuel CLI + 18. Compare network settings after cinder deployment task + 19. Deploy the mongo node using Fuel CLI + 20. Compare network settings after mongo deployment task + 21. Deploy the ceph-osd nodes using Fuel CLI + 22. Compare network settings after ceph-osd deployment task + 23. Deploy the base-os node using Fuel CLI + 24. Compare network settings after base-os deployment task + 25. Check that nodes in discover state stay in it + 26. Get deployment-info + 27. Get cluster settings after deployment task + 28. Compare cluster settings after deploy and before deploy + 29. Run OSTF Duration 50m """ @@ -422,9 +480,10 @@ class CommandLineTest(test_cli_base.CommandLine): node_ids = sorted([node['id'] for node in self.fuel_web.client.list_nodes()]) admin_ip = self.ssh_manager.admin_ip - + self.show_step(5) + cluster_settings = self.download_settings(cluster_id) # Add and provision a controller node node_ids[0] - self.show_step(5, 'on node {0}'.format(node_ids[0])) + self.show_step(6, 'on node {0}'.format(node_ids[0])) cmd = ('fuel --env-id={0} node set --node {1}\ --role=controller'.format(cluster_id, node_ids[0])) @@ -448,7 +507,7 @@ class CommandLineTest(test_cli_base.CommandLine): self.fuel_web.get_nailgun_node_by_status('provisioned'))) # Add and provision 1 compute node_ids[1] - self.show_step(6, details='using node id {}'.format(node_ids[1])) + self.show_step(7, details='using node id {}'.format(node_ids[1])) cmd = ('fuel --env-id={0} node set --node {1}\ --role=compute'.format(cluster_id, node_ids[1])) self.ssh_manager.execute_on_remote(admin_ip, cmd) @@ -472,7 +531,7 @@ class CommandLineTest(test_cli_base.CommandLine): self.fuel_web.get_nailgun_node_by_status('provisioned'))) # Add and provision 1 cinder node_ids[2] - self.show_step(7, details='using node id {}'.format(node_ids[2])) + self.show_step(8, details='using node id {}'.format(node_ids[2])) cmd = ('fuel --env-id={0} node set --node {1}\ --role=cinder'.format(cluster_id, node_ids[2])) self.ssh_manager.execute_on_remote(admin_ip, cmd) @@ -496,7 +555,7 @@ class CommandLineTest(test_cli_base.CommandLine): self.fuel_web.get_nailgun_node_by_status('provisioned'))) # Add and provision 1 mongo node_ids[3] - self.show_step(8, details='using node id {0}'.format(node_ids[3])) + self.show_step(9, details='using node id {0}'.format(node_ids[3])) cmd = ('fuel --env-id={0} node set --node {1}\ --role=mongo'.format(cluster_id, node_ids[3])) self.ssh_manager.execute_on_remote(admin_ip, cmd) @@ -520,7 +579,7 @@ class CommandLineTest(test_cli_base.CommandLine): self.fuel_web.get_nailgun_node_by_status('provisioned'))) # Add and provision 2 ceph-osd node_ids[4], node_ids[5] - self.show_step(9, details='using node ids {0}, {1}'.format( + self.show_step(10, details='using node ids {0}, {1}'.format( node_ids[4], node_ids[5])) cmd = ('fuel --env-id={0} node set --node {1},{2} ' '--role=ceph-osd'.format(cluster_id, node_ids[4], node_ids[5])) @@ -547,7 +606,7 @@ class CommandLineTest(test_cli_base.CommandLine): 'nodes {}'.format( self.fuel_web.get_nailgun_node_by_status('provisioned'))) # Add and provision 1 base-os node node_ids[6] - self.show_step(10, details='using node ids {0}'.format(node_ids[6])) + self.show_step(11, details='using node ids {0}'.format(node_ids[6])) cmd = ('fuel --env-id={0} node set --node {1} ' '--role=base-os'.format(cluster_id, node_ids[6])) self.ssh_manager.execute_on_remote(admin_ip, cmd) @@ -570,7 +629,7 @@ class CommandLineTest(test_cli_base.CommandLine): 'nodes {}'.format( self.fuel_web.get_nailgun_node_by_status('provisioned'))) - self.show_step(11) + self.show_step(12) # Add 2 compute but do not deploy node_ids[7] node_ids[8] cmd = ('fuel --env-id={0} node set --node {1},{2} ' '--role=compute'.format(cluster_id, node_ids[7], node_ids[8])) @@ -589,74 +648,117 @@ class CommandLineTest(test_cli_base.CommandLine): assert_true(node['pending_addition']) # Deploy the controller node node_ids[0] - self.show_step(12, details='for node id {}'.format(node_ids[0])) + self.show_step(13, details='for node id {}'.format(node_ids[0])) cmd = ('fuel --env-id={0} node --deploy --node {1} --json'.format( cluster_id, node_ids[0])) task = self.ssh_manager.execute_on_remote(admin_ip, cmd, jsonify=True)['stdout_json'] + network_settings = self.get_networks(cluster_id) self.assert_cli_task_success(task, timeout=60 * 60) + self.show_step(14) + network_configuration = self.get_net_config_cli(task['id']) + assert_equal(network_settings, + network_configuration, + message='Network settings are not equal before' + ' and after deploy') self.assert_all_tasks_completed(cluster_id=cluster_id) self.show_step(13, details='for node id {}'.format(node_ids[1])) + self.show_step(15, details='for node id {}'.format(node_ids[1])) # Deploy the compute node node_ids[1] cmd = ('fuel --env-id={0} node --deploy --node {1} --json'.format( cluster_id, node_ids[1])) task = self.ssh_manager.execute_on_remote(admin_ip, cmd, jsonify=True)['stdout_json'] + network_settings = self.get_networks(cluster_id) self.assert_cli_task_success(task, timeout=30 * 60) + self.show_step(16) + network_configuration = self.get_net_config_cli(task['id']) + assert_equal(network_settings, + network_configuration, + message='Network settings are not equal before' + ' and after deploy') self.assert_all_tasks_completed(cluster_id=cluster_id) # Deploy the cinder node node_ids[2] - self.show_step(14, details='for node id {}'.format(node_ids[2])) + self.show_step(17, details='for node id {}'.format(node_ids[2])) cmd = ('fuel --env-id={0} node --deploy --node {1} --json'.format( cluster_id, node_ids[2])) task = self.ssh_manager.execute_on_remote(admin_ip, cmd, jsonify=True)['stdout_json'] + network_settings = self.get_networks(cluster_id) self.assert_cli_task_success(task, timeout=60 * 60) + self.show_step(18) + network_configuration = self.get_net_config_cli(task['id']) + assert_equal(network_settings, + network_configuration, + message='Network settings are not equal before' + ' and after deploy') self.assert_all_tasks_completed(cluster_id=cluster_id) # Deploy the mongo node node_ids[3] - self.show_step(15, details='for node id {}'.format(node_ids[3])) + self.show_step(19, details='for node id {}'.format(node_ids[3])) cmd = ('fuel --env-id={0} node --deploy --node {1} --json'.format( cluster_id, node_ids[3])) task = self.ssh_manager.execute_on_remote(admin_ip, cmd, jsonify=True)['stdout_json'] + network_settings = self.get_networks(cluster_id) self.assert_cli_task_success(task, timeout=60 * 60) + self.show_step(20) + network_configuration = self.get_net_config_cli(task['id']) + assert_equal(network_settings, + network_configuration, + message='Network settings are not equal before' + ' and after deploy') self.assert_all_tasks_completed(cluster_id=cluster_id) # Deploy ceph-osd nodes node_ids[4] node_ids[5] - self.show_step(16, details='for node ids {0}, {1}'.format( + self.show_step(21, details='for node ids {0}, {1}'.format( node_ids[4], node_ids[5])) cmd = ('fuel --env-id={0} node --deploy --node {1},{2} --json'.format( cluster_id, node_ids[4], node_ids[5])) task = self.ssh_manager.execute_on_remote(admin_ip, cmd, jsonify=True)['stdout_json'] + network_settings = self.get_networks(cluster_id) self.assert_cli_task_success(task, timeout=60 * 60) + self.show_step(22) + network_configuration = self.get_net_config_cli(task['id']) + assert_equal(network_settings, + network_configuration, + message='Network settings are not equal before' + ' and after deploy') self.assert_all_tasks_completed(cluster_id=cluster_id) # Deploy the base-os node node_ids[6] - self.show_step(17, details='for node id {}'.format(node_ids[6])) + self.show_step(23, details='for node id {}'.format(node_ids[6])) cmd = ('fuel --env-id={0} node --deploy --node {1} --json'.format( cluster_id, node_ids[6])) task = self.ssh_manager.execute_on_remote(admin_ip, cmd, jsonify=True)['stdout_json'] + network_settings = self.get_networks(cluster_id) self.assert_cli_task_success(task, timeout=60 * 60) + self.show_step(24) + network_configuration = self.get_net_config_cli(task['id']) + assert_equal(network_settings, + network_configuration, + message='Network settings are not equal before' + ' and after deploy') self.assert_all_tasks_completed(cluster_id=cluster_id) self.fuel_web.verify_network(cluster_id) - self.show_step(18) + self.show_step(25) node_discover_after_deploy = self.fuel_web.get_nailgun_node_by_status( 'discover') assert_equal( @@ -668,8 +770,18 @@ class CommandLineTest(test_cli_base.CommandLine): for node in node_discover_after_deploy: assert_true(node['pending_addition']) + self.show_step(26) + task_id = self.get_first_task_id_by_name(cluster_id, 'deployment') + self.get_deployment_info_cli(task_id) + self.show_step(27) + cluster_config = self.get_cluster_config_cli(task_id) + self.show_step(28) + assert_equal(cluster_settings, + cluster_config, + message='Cluster settings are not equal before' + ' and after deploy') # Run OSTF - self.show_step(19) + self.show_step(29) self.fuel_web.run_ostf(cluster_id=cluster_id, test_sets=['ha', 'smoke', 'sanity']) self.env.make_snapshot("cli_selected_nodes_deploy_huge") diff --git a/fuelweb_test/tests/test_cli_base.py b/fuelweb_test/tests/test_cli_base.py index 5d9c81952..dc52991ba 100644 --- a/fuelweb_test/tests/test_cli_base.py +++ b/fuelweb_test/tests/test_cli_base.py @@ -48,6 +48,23 @@ class CommandLine(TestBasic): )['stdout_json'] return tasks[0] + @logwrap + def get_tasks(self): + tasks = self.ssh_manager.execute_on_remote( + ip=self.ssh_manager.admin_ip, + cmd='fuel task --json', + jsonify=True)['stdout_json'] + return tasks + + @logwrap + def get_first_task_id_by_name(self, cluster_id, task_name): + tasks = self.get_tasks() + tasks_ids = [] + for task in tasks: + if task['cluster'] == cluster_id and task['name'] == task_name: + tasks_ids.append(task['id']) + return min(tasks_ids) + @logwrap def get_network_filename(self, cluster_id): cmd = ('fuel --env {0} network --download --dir /tmp --json' @@ -411,3 +428,48 @@ class CommandLine(TestBasic): interface['assigned_networks'] = net_to_assign logger.debug("interfaces after update {}".format(interfaces)) self.upload_node_interfaces(node_id, interfaces) + + @logwrap + def get_net_config_cli(self, task_id): + cmd = 'fuel2 task network-configuration download {0}'.format(task_id) + out = self.ssh_manager.execute_on_remote( + ip=self.ssh_manager.admin_ip, + cmd=cmd + )['stdout'] + settings_download = ''.join(out) + settings_file = settings_download.split()[-1] + out = self.ssh_manager.execute_on_remote( + ip=self.ssh_manager.admin_ip, + cmd='cat {0}'.format(settings_file), + yamlify=True)['stdout_yaml'] + return out + + @logwrap + def get_cluster_config_cli(self, task_id): + cmd = 'fuel2 task settings download {0}'.format(task_id) + out = self.ssh_manager.execute_on_remote( + ip=self.ssh_manager.admin_ip, + cmd=cmd + )['stdout'] + settings_download = ''.join(out) + settings_file = settings_download.split()[-1] + out = self.ssh_manager.execute_on_remote( + ip=self.ssh_manager.admin_ip, + cmd='cat {0}'.format(settings_file), + yamlify=True)['stdout_yaml'] + return out + + @logwrap + def get_deployment_info_cli(self, task_id): + cmd = 'fuel2 task deployment-info download {0}'.format(task_id) + out = self.ssh_manager.execute_on_remote( + ip=self.ssh_manager.admin_ip, + cmd=cmd + )['stdout'] + settings_download = ''.join(out) + settings_file = settings_download.split()[-1] + out = self.ssh_manager.execute_on_remote( + ip=self.ssh_manager.admin_ip, + cmd='cat {0}'.format(settings_file), + yamlify=True)['stdout_yaml'] + return out