diff --git a/doc/base_tests.rst b/doc/base_tests.rst index 8391cc447..6fe0aa164 100644 --- a/doc/base_tests.rst +++ b/doc/base_tests.rst @@ -616,6 +616,11 @@ Restart tests Upgrade tests ============= +Test Data-Driven Upgrade +------------------------ +.. automodule:: fuelweb_test.tests.tests_upgrade.test_data_driven_upgrade + :members: + OS upgrade tests ================ diff --git a/fuelweb_test/settings.py b/fuelweb_test/settings.py index 1b53fa1d7..63e6c50f6 100644 --- a/fuelweb_test/settings.py +++ b/fuelweb_test/settings.py @@ -492,6 +492,11 @@ FUEL_PLUGIN_BUILDER_REPO = 'https://github.com/openstack/fuel-plugins.git' # Ubuntu: http://osci-obs.vm.mirantis.net:82/ubuntu-fuel-master-20921/ubuntu/ CUSTOM_PKGS_MIRROR = os.environ.get('CUSTOM_PKGS_MIRROR', '') +# URL for Fuel proposed repository - required for correct ordering of upgrade: +# update Fuel Master node using published packages (UPDATE_FUEL=True) +# then attach the repository in the test case. +FUEL_PROPOSED_REPO_URL = os.environ.get('FUEL_PROPOSED_REPO_URL', '') + # Location of local mirrors on master node. LOCAL_MIRROR_UBUNTU = os.environ.get('LOCAL_MIRROR_UBUNTU', '/var/www/nailgun/ubuntu/x86_64') diff --git a/fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade.py b/fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade.py new file mode 100644 index 000000000..8c1f93365 --- /dev/null +++ b/fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade.py @@ -0,0 +1,1140 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +from devops.helpers.helpers import wait +from proboscis import test +from proboscis.asserts import assert_equal, assert_not_equal +from proboscis.asserts import assert_false +from proboscis.asserts import assert_true + +from fuelweb_test import logger +from fuelweb_test import settings +from fuelweb_test.helpers import checkers +from fuelweb_test.helpers.decorators import log_snapshot_after_test +from fuelweb_test.helpers.utils import run_on_remote +from fuelweb_test.tests.base_test_case import SetupEnvironment +from fuelweb_test.tests.base_test_case import TestBasic + + +class DataDrivenUpgradeBase(TestBasic): + OCTANE_COMMANDS = { + 'backup': 'octane fuel-backup --to {path}', + 'repo-backup': 'octane fuel-repo-backup --to {path}', + 'restore': 'octane fuel-restore --from {path} --admin-password {pwd}', + 'repo-restore': 'octane fuel-repo-restore --from {path}', + 'update-bootstrap-centos': 'octane update-bootstrap-centos' + } + + def __init__(self): + super(DataDrivenUpgradeBase, self).__init__() + self.local_dir_for_backups = settings.LOGS_DIR + self.remote_dir_for_backups = "/root/upgrade/backup" + self.cluster_creds = { + 'tenant': 'upgrade', + 'user': 'upgrade', + 'password': 'upgrade' + } + self.backup_name = None + self.repos_backup_name = None + self.__admin_remote = None + + @property + def backup_path(self): + return os.path.join(self.remote_dir_for_backups, self.backup_name) + + @property + def local_path(self): + return os.path.join(self.local_dir_for_backups, self.backup_name) + + @property + def repos_backup_path(self): + return os.path.join(self.remote_dir_for_backups, + self.repos_backup_name) + + @property + def repos_local_path(self): + return os.path.join(self.local_dir_for_backups, self.repos_backup_name) + + @property + def admin_remote(self): + try: + self.__admin_remote.execute("ls") + # I'm not sure which exception will be raised by paramiko + except Exception as e: + logger.debug( + "Got exception in admin_remote: {!r}\n Reconnecting".format(e) + ) + self.__admin_remote = self.env.d_env.get_admin_remote() + return self.__admin_remote + + @admin_remote.deleter + def admin_remote(self): + if self.__admin_remote: + self.__admin_remote.clear() + + def cleanup(self): + os.remove(self.local_path) + os.remove(self.repos_local_path) + del self.admin_remote + + def install_octane(self): + """ Install fuel-octane package to master node""" + del self.admin_remote + conf_file = None + if settings.FUEL_PROPOSED_REPO_URL: + conf_file = '/etc/yum.repos.d/fuel-proposed.repo' + settings.FUEL_PROPOSED_REPO_URL = os.environ.get( + 'FUEL_PROPOSED_REPO_URL') + cmd = ("echo -e " + "'[fuel-proposed]\n" + "name=fuel-proposed\n" + "baseurl={}/\n" + "gpgcheck=0\n" + "priority=1' > {}").format( + settings.FUEL_PROPOSED_REPO_URL, + conf_file) + + run_on_remote(self.admin_remote, cmd) + + run_on_remote(self.admin_remote, "yum install -y fuel-octane") + + if settings.FUEL_PROPOSED_REPO_URL: + # pylint: disable=no-member + self.admin_remote.rm_rf(conf_file) + # pylint: enable=no-member + + def octane_action(self, action, path=None): + assert_true(action in self.OCTANE_COMMANDS.keys(), + "Unknown octane action '{}', aborting".format(action)) + octane_cli_args = { + 'path': path, + 'pwd': settings.KEYSTONE_CREDS['password'] + } + if 'backup' in action: + # pylint: disable=no-member + assert_false(self.admin_remote.exists(path), + 'File already exists, not able to reuse') + # pylint: enable=no-member + elif 'restore' in action: + checkers.check_file_exists(self.admin_remote, path) + + run_on_remote(self.admin_remote, + self.OCTANE_COMMANDS[action].format(**octane_cli_args)) + + if 'backup' in action: + checkers.check_file_exists(self.admin_remote, path) + + def do_backup(self, + backup_path, local_path, + repos_backup_path=None, repos_local_path=None): + """ Wrapper for backup process of upgrading procedure""" + # BOTH repos arguments should be passed at the same time + # or BOTH should not be passed + assert_equal(bool(repos_backup_path), bool(repos_local_path), + "Both repos arguments should be specified") + self.install_octane() + + cmd = "mkdir -p {}".format(self.remote_dir_for_backups) + run_on_remote(self.admin_remote, cmd) + + self.octane_action("backup", backup_path) + logger.info("Downloading {}".format(backup_path)) + # pylint: disable=no-member + self.admin_remote.download(backup_path, local_path) + # pylint: enable=no-member + assert_true(os.path.exists(local_path)) + + if repos_backup_path: + self.octane_action("repo-backup", repos_backup_path) + logger.info("Downloading {}".format(repos_backup_path)) + # pylint: disable=no-member + self.admin_remote.download(repos_backup_path, repos_local_path) + # pylint: enable=no-member + assert_true(os.path.exists(repos_local_path)) + + def do_restore(self, + backup_path, local_path, + repos_backup_path=None, repos_local_path=None): + """ Wrapper for restore process of upgrading procedure""" + # BOTH repos arguments should be passed at the same time + # or BOTH should not be passed + assert_equal(bool(repos_backup_path), bool(repos_local_path), + "Both repos arguments should be specified") + self.install_octane() + + cmd = "mkdir -p {}".format(self.remote_dir_for_backups) + run_on_remote(self.admin_remote, cmd) + + logger.info("Uploading {}".format(local_path)) + # pylint: disable=no-member + self.admin_remote.upload(local_path, backup_path) + # pylint: enable=no-member + logger.info("Applying backup from {}".format(backup_path)) + self.octane_action("restore", backup_path) + + if repos_backup_path: + logger.info("Uploading {}".format(repos_local_path)) + # pylint: disable=no-member + self.admin_remote.upload(repos_local_path, repos_backup_path) + # pylint: enable=no-member + logger.info("Applying backup from {}".format(repos_backup_path)) + self.octane_action("repo-restore", repos_backup_path) + + logger.info( + "Update existing CentOS bootstrap image using restored ssh keys") + self.octane_action('update-bootstrap-centos') + + n_nodes = self.fuel_web.client.list_nodes() + d_nodes = self.fuel_web.get_devops_nodes_by_nailgun_nodes(n_nodes) + discover_n_nodes = [node for node in self.fuel_web.client.list_nodes() + if self.fuel_web.is_node_discovered(node)] + + if discover_n_nodes: + logger.info("Rebooting bootstrapped nodes") + discover_d_nodes = self.fuel_web.get_devops_nodes_by_nailgun_nodes( + discover_n_nodes) + self.fuel_web.cold_restart_nodes(discover_d_nodes) + + # LP: 1561092 mcollective can stuck after upgrade + logger.info("Applying fix for LP:1561092") + for node in d_nodes: + with self.fuel_web.get_ssh_for_node(node_name=node.name) as remote: + run_on_remote(remote, "service mcollective restart") + + def deploy_cluster(self, cluster_settings): + slaves_count = len(cluster_settings['nodes']) + slaves = self.env.d_env.nodes().slaves[:slaves_count] + for chunk in [slaves[x:x + 5] for x in range(0, slaves_count, 5)]: + self.env.bootstrap_nodes(chunk) + cluster_id = self.fuel_web.create_cluster( + name=cluster_settings['name'], + mode=settings.DEPLOYMENT_MODE, + settings=cluster_settings['settings'] + ) + if cluster_settings.get('plugin'): + plugin_name = cluster_settings['plugin']['name'] + assert_true( + self.fuel_web.check_plugin_exists(cluster_id, plugin_name)) + self.fuel_web.update_plugin_data( + cluster_id, plugin_name, cluster_settings['plugin']['data']) + + self.fuel_web.update_nodes(cluster_id, cluster_settings['nodes']) + self.fuel_web.verify_network(cluster_id) + + # Code for debugging on hosts with low IO + # for chunk in [slaves[x:x+5] for x in range(0, slaves_count, 5)]: + # ids = [self.fuel_web.get_nailgun_node_by_devops_node(x)['id'] + # for x in chunk] + # self.fuel_web.client.provision_nodes(cluster_id, ids) + # wait(lambda: all( + # [self.fuel_web.get_nailgun_node_by_devops_node(node)['status' + # ] == 'provisioned' for node in chunk]), + # timeout=30 * 60, + # interval=60) + + self.fuel_web.deploy_cluster_wait(cluster_id) + self.fuel_web.verify_network(cluster_id) + self.fuel_web.run_ostf(cluster_id) + + def prepare_upgrade_smoke(self): + self.backup_name = "backup_smoke.tar.gz" + self.repos_backup_name = "repos_backup_smoke.tar.gz" + + self.check_run("upgrade_smoke_backup") + self.env.revert_snapshot("ready", skip_timesync=True) + intermediate_snapshot = "prepare_upgrade_smoke_before_backup" + + assert_not_equal( + settings.KEYSTONE_CREDS['password'], 'admin', + "Admin password was not changed, aborting execution") + + cluster_settings = { + 'net_provider': settings.NEUTRON, + 'net_segment_type': settings.NEUTRON_SEGMENT['vlan'] + } + cluster_settings.update(self.cluster_creds) + + if not self.env.d_env.has_snapshot(intermediate_snapshot): + self.deploy_cluster( + {'name': self.prepare_upgrade_smoke.__name__, + 'settings': cluster_settings, + 'nodes': {'slave-01': ['controller'], + 'slave-02': ['compute', 'cinder']} + } + ) + self.env.make_snapshot(intermediate_snapshot) + + # revert_snapshot will do nothing if there is no snapshot + self.env.revert_snapshot(intermediate_snapshot) + + self.do_backup(self.backup_path, self.local_path, + self.repos_backup_path, self.repos_local_path) + self.env.make_snapshot("upgrade_smoke_backup", is_make=True) + + def prepare_upgrade_ceph_ha(self): + self.backup_name = "backup_ceph_ha.tar.gz" + self.repos_backup_name = "repos_backup_ceph_ha.tar.gz" + + self.check_run("upgrade_ceph_ha_backup") + self.env.revert_snapshot("ready", skip_timesync=True) + intermediate_snapshot = "prepare_upgrade_ceph_ha_before_backup" + + assert_not_equal( + settings.KEYSTONE_CREDS['password'], 'admin', + "Admin password was not changed, aborting execution") + + cluster_settings = { + 'net_provider': settings.NEUTRON, + 'net_segment_type': settings.NEUTRON_SEGMENT['tun'], + 'volumes_lvm': False, + 'volumes_ceph': True, + 'images_ceph': True, + 'objects_ceph': True, + 'ephemeral_ceph': True, + } + cluster_settings.update(self.cluster_creds) + + if not self.env.d_env.has_snapshot(intermediate_snapshot): + self.deploy_cluster( + {'name': self.prepare_upgrade_ceph_ha.__name__, + 'settings': cluster_settings, + 'nodes': + {'slave-01': ['controller'], + 'slave-02': ['controller'], + 'slave-03': ['controller'], + 'slave-04': ['compute', 'ceph-osd'], + 'slave-05': ['compute', 'ceph-osd']} + } + ) + self.env.make_snapshot(intermediate_snapshot) + + self.env.revert_snapshot(intermediate_snapshot) + + self.do_backup(self.backup_path, self.local_path, + self.repos_backup_path, self.repos_local_path) + + self.env.make_snapshot("upgrade_ceph_ha_backup", is_make=True) + + def prepare_upgrade_detach_plugin(self): + self.backup_name = "backup_detach_plugin.tar.gz" + self.repos_backup_name = "repos_backup_detach_plugin.tar.gz" + + self.check_run("upgrade_detach_plugin_backup") + self.env.revert_snapshot("ready", skip_timesync=True) + + run_on_remote( + self.admin_remote, + "yum -y install git python-pip createrepo dpkg-devel dpkg-dev rpm " + "rpm-build && pip install fuel-plugin-builder") + + run_on_remote( + self.admin_remote, + "git clone https://github.com/" + "openstack/fuel-plugin-detach-database") + + cmds = [ + "cd fuel-plugin-detach-database", "git checkout stable/{}".format( + settings.UPGRADE_FUEL_FROM), + "fpb --build . ", + "fuel plugins --install *.rpm " + "--user {user} --password {pwd}".format( + user=settings.KEYSTONE_CREDS['username'], + pwd=settings.KEYSTONE_CREDS['password']) + ] + + run_on_remote(self.admin_remote, " && ".join(cmds)) + + cluster_settings = { + 'net_provider': settings.NEUTRON, + 'net_segment_type': settings.NEUTRON_SEGMENT['tun'], + 'volumes_lvm': False, + 'volumes_ceph': True, + 'images_ceph': True, + 'objects_ceph': True, + 'ephemeral_ceph': True, + } + cluster_settings.update(self.cluster_creds) + + self.deploy_cluster({ + 'name': self.prepare_upgrade_detach_plugin.__name__, + 'settings': cluster_settings, + 'plugin': + {'name': 'detach-database', + 'data': {'metadata/enabled': True}}, + 'nodes': + {'slave-01': ['controller'], + 'slave-02': ['controller'], + 'slave-03': ['controller'], + 'slave-04': ['standalone-database'], + 'slave-05': ['standalone-database'], + 'slave-06': ['standalone-database'], + 'slave-07': ['compute', 'ceph-osd'], + 'slave-08': ['compute', 'ceph-osd']} + }) + + self.do_backup(self.backup_path, self.local_path, + self.repos_backup_path, self.repos_local_path) + self.env.make_snapshot("upgrade_detach_plugin_backup", is_make=True) + + +@test +class UpgradePrepare(DataDrivenUpgradeBase): + """Base class for initial preparation of 7.0 env and clusters.""" + + cluster_creds = { + 'tenant': 'upgrade', + 'user': 'upgrade', + 'password': 'upgrade' + } + + @test(groups=['upgrade_smoke_backup'], + depends_on=[SetupEnvironment.prepare_release]) + @log_snapshot_after_test + def upgrade_smoke_backup(self): + """Prepare non-HA+cinder cluster using previous version of Fuel + Nailgun password should be changed via KEYSTONE_PASSWORD env variable + + Scenario: + 1. Create cluster with default configuration + 2. Add 1 node with controller role + 3. Add 1 node with compute+cinder roles + 4. Verify networks + 5. Deploy cluster + 6. Run OSTF + 7. Install fuel-octane package + 8. Create backup file using 'octane fuel-backup' + 9. Download the backup to the host + + Duration: TODO + Snapshot: upgrade_smoke_backup + """ + super(self.__class__, self).prepare_upgrade_smoke() + + @test(groups=['upgrade_ceph_ha_backup'], + depends_on=[SetupEnvironment.prepare_release]) + @log_snapshot_after_test + def upgrade_ceph_ha_backup(self): + """Prepare HA, ceph for all cluster using previous version of Fuel. + Nailgun password should be changed via KEYSTONE_PASSWORD env variable + + Scenario: + 1. Create cluster with NeutronTUN and ceph for all + 2. Add 3 node with controller role + 3. Add 2 node with compute+ceph roles + 4. Verify networks + 5. Deploy cluster + 6. Run OSTF + 7. Install fuel-octane package + 8. Create backup file using 'octane fuel-backup' + 9. Download the backup to the host + + Duration: TODO + Snapshot: upgrade_ceph_ha_backup + """ + + super(self.__class__, self).prepare_upgrade_ceph_ha() + + @test(groups=['upgrade_detach_plugin_backup'], + depends_on=[SetupEnvironment.prepare_slaves_9]) + @log_snapshot_after_test + def upgrade_detach_plugin_backup(self): + """Initial preparation of the cluster using previous version of Fuel; + Using: HA, ceph for all + + Scenario: + 1. Install detach-database plugin on master node + 2. Create cluster with default configuration + 3. Enable plugin for created cluster + 4. Add 3 node with controller role + 5. Add 3 node with separate-database role + 6. Add 2 node with compute+ceph roles + 7. Verify networks + 8. Deploy cluster + 9. Run OSTF + 10. Install fuel-octane package + 11. Create backup file using 'octane fuel-backup' + 12. Download the backup to the host + + Duration: TODO + Snapshot: upgrade_detach_plugin_backup + """ + super(self.__class__, self).prepare_upgrade_detach_plugin() + + +@test(groups=['upgrade_rollback_tests']) +class UpgradeRollback(DataDrivenUpgradeBase): + def __init__(self): + super(UpgradeRollback, self).__init__() + self.backup_name = "backup_ceph_ha.tar.gz" + self.repos_backup_name = "repos_backup_ceph_ha.tar.gz" + self.source_snapshot_name = "upgrade_ceph_ha_backup" + self.snapshot_name = "upgrade_rollback_ceph_ha" + + @test(groups=['upgrade_rollback_ceph_ha'], + depends_on=[UpgradePrepare.upgrade_ceph_ha_backup]) + @log_snapshot_after_test + def upgrade_rollback_ceph_ha(self): + """Restore 7.0 Fuel with ha cluster using octane + + Scenario: + 1. Revert "upgrade_ceph_ha_backup" snapshot. + 2. Reinstall Fuel master node as usual. + 3. Restore previously backup-ed data using fuel-octane. + 4. Validate that data was restored using nailgun api. + 5. Validate that node can be bootstrapped. + + Snapshot: upgrade_rollback_ceph_ha + Duration: XX m + """ + assert_true(os.path.exists(self.repos_local_path)) + assert_true(os.path.exists(self.local_path)) + self.check_run(self.snapshot_name) + self.show_step(1) + assert_true( + self.env.revert_snapshot(self.source_snapshot_name), + "The test can not use given environment - snapshot " + "{!r} does not exists".format(self.source_snapshot_name)) + self.show_step(2) + old_cluster_id = self.fuel_web.get_last_created_cluster() + self.env.reinstall_master_node() + self.show_step(3) + self.do_restore(self.backup_path, self.local_path, + self.repos_backup_path, self.repos_local_path) + self.show_step(4) + cluster_id = self.fuel_web.get_last_created_cluster() + assert_equal(old_cluster_id, cluster_id, + "Cluster IDs are mismatch after upgrade") + creds = self.fuel_web.get_cluster_credentials(cluster_id) + assert_equal( + sorted(creds.values()), + sorted(self.cluster_creds.values()) + ) + self.show_step(5) + slave_06 = self.env.d_env.get_node(name="slave-06") + self.env.bootstrap_nodes([slave_06]) + self.env.make_snapshot(self.snapshot_name, is_make=True) + self.cleanup() + + @test(groups=["upgrade_rollback_ceph_ha_scale"], + depends_on=[upgrade_rollback_ceph_ha]) + @log_snapshot_after_test + def upgrade_rollback_ceph_ha_scale(self): + """Scale cluster after rollback + + Scenario: + 1. Revert "upgrade_rollback_ceph_ha" snapshot. + 2. Add 1 controller to existing cluster. + 3. Deploy changes. + 4. Verify networks. + 5. Run OSTF + + Duration: TODO + """ + self.show_step(1) + self.env.revert_snapshot(self.snapshot_name) + + self.show_step(2) + cluster_id = self.fuel_web.get_last_created_cluster() + self.fuel_web.update_nodes(cluster_id, {'slave-06': ['controller']}) + self.show_step(3) + self.fuel_web.deploy_cluster_wait(cluster_id) + + self.show_step(4) + self.fuel_web.verify_network(cluster_id) + self.show_step(5) + self.fuel_web.run_ostf(cluster_id) + + @test(groups=["upgrade_rollback_reset_redeploy"], + depends_on=[upgrade_rollback_ceph_ha]) + @log_snapshot_after_test + def upgrade_rollback_reset_redeploy(self): + """After rollback reset existing cluster and redeploy + + Scenario: + 1. Revert "upgrade_rollback_ceph_ha" snapshot. + 2. Reset cluster and wait until nodes are bootstraped. + 3. Deploy changes. + 4. Verify networks. + 5. Run OSTF. + + Duration: TODO + """ + self.show_step(1) + self.env.revert_snapshot(self.snapshot_name, skip_timesync=True) + + self.show_step(2) + cluster_id = self.fuel_web.get_last_created_cluster() + self.fuel_web.stop_reset_env_wait(cluster_id) + + self.fuel_web.wait_nodes_get_online_state( + self.env.d_env.nodes().slaves[:5], timeout=10 * 60) + + self.show_step(3) + self.fuel_web.deploy_cluster_wait(cluster_id) + self.show_step(4) + self.fuel_web.verify_network(cluster_id) + self.show_step(5) + self.fuel_web.run_ostf(cluster_id) + + @test(groups=["upgrade_rollback_new_deploy"], + depends_on=[upgrade_rollback_ceph_ha]) + @log_snapshot_after_test + def upgrade_rollback_new_deploy(self): + """After rollback delete existing cluster and deploy new one, + + Scenario: + 1. Revert "upgrade_rollback_ceph_ha" snapshot. + 2. Delete cluster and wait until nodes are bootstraped. + 3. Create new cluster with NeutronVLAN + Ceph. + 4. Add 3 controllers. + 5. Add 2 compute + ceph nodes. + 6. Deploy cluster. + 7. Verify networks. + 8. Run OSTF. + + Duration: TODO + """ + self.show_step(1) + self.env.revert_snapshot(self.snapshot_name, skip_timesync=True) + + self.show_step(2) + cluster_id = self.fuel_web.get_last_created_cluster() + devops_nodes = self.fuel_web.get_devops_nodes_by_nailgun_nodes( + self.fuel_web.client.list_cluster_nodes(cluster_id=cluster_id)) + self.fuel_web.client.delete_cluster(cluster_id) + wait(lambda: not any([cluster['id'] == cluster_id for cluster in + self.fuel_web.client.list_clusters()]), + timeout=60 * 10) + self.env.bootstrap_nodes(devops_nodes) + + self.show_step(3) + cluster_settings = { + 'net_provider': settings.NEUTRON, + 'net_segment_type': settings.NEUTRON_SEGMENT['vlan'], + 'volumes_lvm': False, + 'volumes_ceph': True, + 'images_ceph': True, + 'objects_ceph': True, + 'ephemeral_ceph': True, + } + cluster_settings.update(self.cluster_creds) + + cluster_id = self.fuel_web.create_cluster( + name=self.upgrade_rollback_new_deploy.__name__, + mode=settings.DEPLOYMENT_MODE, + settings=cluster_settings) + + self.show_step(4) + self.show_step(5) + self.fuel_web.update_nodes( + cluster_id, + {'slave-01': ['controller'], + 'slave-02': ['controller'], + 'slave-03': ['controller'], + 'slave-04': ['compute', 'ceph-osd'], + 'slave-05': ['compute', 'ceph-osd']}) + self.show_step(6) + self.fuel_web.deploy_cluster_wait(cluster_id) + self.show_step(7) + self.fuel_web.verify_network(cluster_id) + self.show_step(8) + self.fuel_web.run_ostf(cluster_id) + + +@test(groups=['upgrade_smoke_tests']) +class UpgradeSmoke(DataDrivenUpgradeBase): + + def __init__(self): + super(UpgradeSmoke, self).__init__() + self.backup_name = "backup_smoke.tar.gz" + self.repos_backup_name = "repos_backup_smoke.tar.gz" + self.source_snapshot_name = "upgrade_smoke_backup" + self.snapshot_name = "upgrade_smoke_restore" + + @test(groups=['upgrade_smoke_restore']) + @log_snapshot_after_test + def upgrade_smoke_restore(self): + """Reinstall Fuel and restore non-HA cluster using fuel-octane. + + Scenario: + 1. Revert "upgrade_smoke" snapshot + 2. Reinstall Fuel master using iso given in ISO_PATH + 3. Install fuel-octane package + 4. Upload the backup back to reinstalled Fuel maser node + 5. Restore master node using 'octane fuel-restore' + 6. Check that nailgun is available + 7. Check cobbler configs for all discovered nodes + 8. Check ubuntu bootstrap is available + 9. Verify networks + 10. Run OSTF + + Snapshot: upgrade_smoke_restore + Duration: TODO + """ + + assert_true(os.path.exists(self.repos_local_path)) + assert_true(os.path.exists(self.local_path)) + + self.check_run(self.snapshot_name) + self.show_step(1, initialize=True) + assert_true( + self.env.revert_snapshot(self.source_snapshot_name), + "The test can not use given environment - snapshot " + "{!r} does not exists".format(self.source_snapshot_name)) + self.show_step(2) + self.env.reinstall_master_node() + self.show_step(3) + self.show_step(4) + self.show_step(5) + self.do_restore(self.backup_path, self.local_path, + self.repos_backup_path, self.repos_local_path) + # Check nailgun api is available + self.show_step(6) + self.fuel_web.change_default_network_settings() + + cluster_id = self.fuel_web.get_last_created_cluster() + + self.show_step(7) + for node in self.fuel_web.client.list_cluster_nodes(cluster_id): + checkers.check_cobbler_node_exists(self.admin_remote, node['id']) + + # Check non-default parameters of the cluster + creds = self.fuel_web.get_cluster_credentials(cluster_id) + assert_equal(sorted(creds.values()), + sorted(self.cluster_creds.values())) + + self.show_step(8) + slave_03 = self.env.d_env.get_node(name="slave-03") + self.env.bootstrap_nodes([slave_03]) + with self.fuel_web.get_ssh_for_node(slave_03.name) as slave_remote: + checkers.verify_bootstrap_on_node(slave_remote, "ubuntu") + + self.show_step(9) + self.fuel_web.verify_network(cluster_id) + self.show_step(10) + self.fuel_web.run_ostf(cluster_id) + + self.env.make_snapshot("upgrade_smoke_restore", is_make=True) + self.cleanup() + + @test(groups=['upgrade_smoke_scale'], + depends_on=[upgrade_smoke_restore]) + @log_snapshot_after_test + def upgrade_smoke_scale(self): + """Scale already existing Kilo cluster using upgraded to 8.0 Fuel. + + Scenario: + 1. Revert 'upgrade_smoke_restore' snapshot + 2. Add to existing cluster 3 nodes with controller role + 3. Add to existing cluster 1 node with compute+cinder roles + 4. Verify network + 5. Deploy changes + 6. Run OSTF + 7. Remove from the cluster 1 node with controller role + 8. Remove from the cluster 1 node with compute+cinder roles + 9. Deploy changes + 10. Wait until nodes are discovered + 11. Verify that bootstrapped nodes are using ubuntu bootstrap + 12. Verify network + 13. Run OSTF + + Snapshot: upgrade_smoke_scale + Duration: TODO + """ + self.show_step(1, initialize=True) + self.env.revert_snapshot(self.snapshot_name) + + self.show_step(2) + self.show_step(3) + cluster_id = self.fuel_web.get_last_created_cluster() + self.env.bootstrap_nodes(self.env.d_env.nodes().slaves[3:6]) + self.fuel_web.update_nodes( + cluster_id, + {'slave-04': ['controller'], + 'slave-05': ['controller'], + 'slave-06': ['controller']}) + self.show_step(4) + self.fuel_web.verify_network(cluster_id) + self.show_step(5) + # LP 1562736 get_devops_node_by_nailgun_node is not working + self.fuel_web.deploy_cluster_wait(cluster_id) + self.show_step(6) + self.fuel_web.run_ostf(cluster_id) + + cluster_id = self.fuel_web.get_last_created_cluster() + self.fuel_web.deploy_cluster_wait(cluster_id) + self.show_step(7) + self.show_step(8) + nodes_to_remove = {'slave-06': ['controller']} + + nailgun_nodes = self.fuel_web.update_nodes( + cluster_id, nodes_to_remove, False, True) + + pending_nodes = [x for x in nailgun_nodes if x["pending_deletion"]] + self.show_step(9) + self.fuel_web.deploy_cluster_wait(cluster_id, check_services=False) + self.show_step(10) + self.show_step(11) + for node in pending_nodes: + wait(lambda: self.fuel_web.is_node_discovered(node), + timeout=6 * 60) + with self.fuel_web.get_ssh_for_node( + self.fuel_web.get_devops_node_by_nailgun_node( + node).name) as slave_remote: + checkers.verify_bootstrap_on_node(slave_remote, "ubuntu") + self.show_step(12) + self.fuel_web.verify_network(cluster_id) + self.show_step(13) + self.fuel_web.run_ostf(cluster_id) + self.env.make_snapshot("upgrade_smoke_scale") + + @test(groups=['upgrade_smoke_reset_deploy'], + depends_on=[upgrade_smoke_restore]) + @log_snapshot_after_test + def upgrade_smore_reset_deploy(self): + """Reset existing cluster 7.0 cluster and redeploy + + Scenario: + 1. Revert "upgrade_smoke_restore". + 2. Reset cluster. + 3. Delete nodes from nailgun. + 4. Wait until nodes are discovered. + 5. Re-add nodes back to cluster. + 6. Verify networks. + 7. Deploy cluster. + 8. Run OSTF. + + Duration: TODO + """ + self.show_step(1, initialize=True) + self.env.revert_snapshot("upgrade_smoke_restore") + + self.show_step(2) + cluster_id = self.fuel_web.get_last_created_cluster() + self.fuel_web.stop_reset_env_wait(cluster_id) + + # After reset nodes will use new interface naming scheme which + # conflicts with nailgun data (it still contains eth-named + # interfaces and there is no way to fix it) + # LP : 1553210 + self.show_step(3) + for node in self.fuel_web.client.list_cluster_nodes( + cluster_id=cluster_id): + self.fuel_web.delete_node(node['id']) + + self.show_step(4) + slaves = self.env.d_env.nodes().slaves[:2] + wait(lambda: all(self.env.nailgun_nodes(slaves)), timeout=10 * 60) + for node in self.fuel_web.client.list_cluster_nodes( + cluster_id=cluster_id): + wait(lambda: self.fuel_web.is_node_discovered(node), timeout=60) + + self.show_step(5) + self.fuel_web.update_nodes( + cluster_id, + { + 'slave-01': ['controller'], + 'slave-02': ['compute', 'cinder'] + } + ) + self.show_step(6) + self.fuel_web.verify_network(cluster_id) + self.show_step(7) + self.fuel_web.deploy_cluster_wait(cluster_id) + self.show_step(8) + self.fuel_web.run_ostf(cluster_id) + + @test(groups=['upgrade_smoke_new_deployment'], + depends_on=[upgrade_smoke_restore]) + @log_snapshot_after_test + def upgrade_smoke_new_deployment(self): + """Deploy Liberty cluster using upgraded to 8.0 Fuel. + + Scenario: + 1. Revert 'upgrade_smoke_restore' snapshot + 2. Delete existing cluster + 3. Create new cluster with default parameters + 4. Add 1 node with controller role + 5. Add 1 node with compute+cinder roles + 6. Verify network + 7. Deploy changes + 8. Run OSTF + + Snapshot: upgrade_smoke_new_deployment + Duration: TODO + """ + self.show_step(1, initialize=True) + self.env.revert_snapshot("upgrade_smoke_restore", skip_timesync=True) + + self.show_step(2) + cluster_id = self.fuel_web.get_last_created_cluster() + devops_nodes = self.fuel_web.get_devops_nodes_by_nailgun_nodes( + self.fuel_web.client.list_cluster_nodes(cluster_id=cluster_id) + ) + self.fuel_web.client.delete_cluster(cluster_id) + wait(lambda: not any([cluster['id'] == cluster_id for cluster in + self.fuel_web.client.list_clusters()])) + self.env.bootstrap_nodes(devops_nodes) + + self.show_step(3) + releases = self.fuel_web.client.get_releases() + release_id = [ + release['id'] for release in releases if + release['is_deployable'] and + settings.UPGRADE_FUEL_TO in release['version']][0] + cluster_id = self.fuel_web.create_cluster( + name=self.upgrade_smoke_new_deployment.__name__, + mode=settings.DEPLOYMENT_MODE, + release_id=release_id, + settings={ + 'net_provider': settings.NEUTRON, + 'net_segment_type': settings.NEUTRON_SEGMENT['vlan'] + } + ) + self.show_step(4) + self.show_step(5) + self.fuel_web.update_nodes( + cluster_id, + { + 'slave-01': ['controller'], + 'slave-02': ['compute', 'cinder'] + } + ) + self.show_step(6) + self.fuel_web.verify_network(cluster_id) + self.show_step(7) + self.fuel_web.deploy_cluster_wait(cluster_id) + self.show_step(8) + self.fuel_web.run_ostf(cluster_id) + + +@test(groups=['upgrade_ceph_ha_tests']) +class UpgradeCephHA(DataDrivenUpgradeBase): + def __init__(self): + super(UpgradeCephHA, self).__init__() + self.source_snapshot_name = "upgrade_ceph_ha_backup" + self.snapshot_name = "upgrade_ceph_ha_restore" + self.backup_name = "backup_ceph_ha.tar.gz" + self.repos_backup_name = "repos_backup_ceph_ha.tar.gz" + + @log_snapshot_after_test + @test(groups=['upgrade_ceph_ha_restore']) + def upgrade_ceph_ha_restore(self): + """Reinstall Fuel and restore data with Tun+Ceph+HA cluster + + Scenario: + 1. Revert "upgrade_ceph_ha_backup" snapshot + 2. Reinstall Fuel master using iso given in ISO_PATH + 3. Install fuel-octane package + 4. Upload the backup back to reinstalled Fuel maser node + 5. Restore master node using 'octane fuel-restore' + 6. Verify networks for restored cluster + 7. Run OSTF for restored cluster + + Snapshot: upgrade_ceph_ha_restore + Duration: TODO + """ + assert_true(os.path.exists(self.repos_local_path)) + assert_true(os.path.exists(self.local_path)) + + self.check_run(self.snapshot_name) + self.show_step(1, initialize=True) + assert_true( + self.env.revert_snapshot(self.source_snapshot_name), + "The test can not use given environment - snapshot " + "'upgrade_ceph_ha_backup' does not exists") + + cluster_id = self.fuel_web.get_last_created_cluster() + + self.show_step(2) + self.env.reinstall_master_node() + self.show_step(3) + self.show_step(4) + self.show_step(5) + self.do_restore(self.backup_path, self.local_path, + self.repos_backup_path, self.repos_local_path) + self.fuel_web.change_default_network_settings() + self.env.sync_time() + + self.show_step(6) + self.fuel_web.verify_network(cluster_id) + self.show_step(7) + self.fuel_web.run_ostf(cluster_id) + + self.env.make_snapshot(self.snapshot_name, is_make=True) + self.cleanup() + + @test(groups=['upgrade_ceph_ha_reboot_ctrl'], + depends_on=[upgrade_ceph_ha_restore]) + @log_snapshot_after_test + def upgrade_ceph_ha_reboot_ctrl(self): + """Ensure that controller receives correct boot order from cobbler + + Scenario: + 1. Revert "upgrade_ceph_ha_restore" snapshot. + 2. Warm restart of a controller. + 3. Wait until HA services become ready. + 4. Run OSTF. + + Duration: 20m + """ + self.show_step(1) + self.env.revert_snapshot(self.snapshot_name) + self.show_step(2) + cluster_id = self.fuel_web.get_last_created_cluster() + + n_ctrls = self.fuel_web.get_nailgun_cluster_nodes_by_roles( + cluster_id=cluster_id, + roles=['controller']) + d_ctrls = self.fuel_web.get_devops_nodes_by_nailgun_nodes(n_ctrls) + self.fuel_web.warm_restart_nodes([d_ctrls[0]]) + self.show_step(3) + self.fuel_web.assert_ha_services_ready(cluster_id) + self.show_step(4) + self.fuel_web.run_ostf(cluster_id) + + @test(groups=['upgrade_ceph_ha_scale_ceph'], + depends_on=[upgrade_ceph_ha_restore]) + @log_snapshot_after_test + def upgrade_ceph_ha_scale_ceph(self): + """ Add 1 ceph node to existing cluster after upgrade + + Scenario: + 1. Revert "upgrade_ceph_ha_restore" snapshot. + 2. Add 1 ceph node + 3. Verify networks + 4. Deploy cluster + 5. Run OSTF + + """ + self.show_step(1, initialize=True) + self.env.revert_snapshot(self.snapshot_name) + + self.show_step(2) + cluster_id = self.fuel_web.get_last_created_cluster() + self.env.bootstrap_nodes(self.env.d_env.nodes().slaves[5:6]) + self.fuel_web.update_nodes(cluster_id, {'slave-06': ['ceph-osd']}) + self.show_step(3) + self.fuel_web.verify_network(cluster_id) + self.show_step(4) + # LP 1562736 get_devops_node_by_nailgun_node is not working + self.fuel_web.deploy_cluster_wait(cluster_id) + self.show_step(5) + self.fuel_web.run_ostf(cluster_id) + + +@test(groups=['upgrade_detach_plugin_tests']) +class UpgradeDetach_Plugin(DataDrivenUpgradeBase): + def __init__(self): + super(self.__class__, self).__init__() + self.source_snapshot_name = "upgrade_detach_plugin_backup" + self.snapshot_name = "upgrade_detach_plugin_restore" + self.backup_name = "backup_detach_plugin.tar.gz" + self.repos_backup_name = "repos_backup_detach_plugin.tar.gz" + + @log_snapshot_after_test + @test(groups=['upgrade_detach_plugin_restore']) + def upgrade_detach_plugin_restore(self): + """Reinstall Fuel and restore data with cluster with detach-db plugin + + Scenario: + 1. Revert "upgrade_detach_plugin_backup" snapshot + 2. Reinstall Fuel master using iso given in ISO_PATH + 3. Install fuel-octane package + 4. Upload the backup back to reinstalled Fuel maser node + 5. Restore master node using 'octane fuel-restore' + 6. Ensure that plugin were restored + 7. Verify networks for restored cluster + 8. Run OSTF for restored cluster + + Snapshot: upgrade_detach_plugin_restore + Duration: TODO + """ + assert_true(os.path.exists(self.repos_local_path)) + assert_true(os.path.exists(self.local_path)) + + self.check_run(self.snapshot_name) + self.show_step(1, initialize=True) + assert_true( + self.env.revert_snapshot(self.source_snapshot_name), + "The test can not use given environment - snapshot " + "{!r} does not exists".format(self.source_snapshot_name)) + + cluster_id = self.fuel_web.get_last_created_cluster() + + self.show_step(2) + self.env.reinstall_master_node() + self.show_step(3) + self.show_step(4) + self.show_step(5) + self.do_restore(self.backup_path, self.local_path, + self.repos_backup_path, self.repos_local_path) + self.fuel_web.change_default_network_settings() + self.env.sync_time() + + self.show_step(6) + attr = self.fuel_web.client.get_cluster_attributes(cluster_id) + assert_true('detach-database' in attr['editable'], + "Can't find plugin data in cluster attributes!") + stdout = run_on_remote( + self.admin_remote, + "find /var/www/nailgun/plugins/ -name detach-database*") + assert_not_equal(len(stdout), 0, "Can not find plugin's directory") + plugin_dir = stdout[0].strip() + + checkers.check_file_exists(self.admin_remote, + os.path.join(plugin_dir, "metadata.yaml")) + + self.show_step(7) + self.fuel_web.verify_network(cluster_id) + self.show_step(8) + self.fuel_web.run_ostf(cluster_id) + + self.env.make_snapshot(self.snapshot_name, is_make=True) + self.cleanup() + + @log_snapshot_after_test + @test(groups=['upgrade_detach_plugin_scale'], + depends_on=[upgrade_detach_plugin_restore]) + def upgrade_detach_plugin_scale(self): + """Add 1 node with plugin custom role to existing cluster + + Scenario: + 1. Revert "upgrade_detach_plugin_backup" snapshot. + 2. Add 1 separate-database node + 3. Verify networks + 4. Deploy cluster + 5. Run OSTF + + Duration: 60m + """ + + self.show_step(1, initialize=True) + self.env.revert_snapshot(self.snapshot_name) + + self.show_step(2) + cluster_id = self.fuel_web.get_last_created_cluster() + self.env.bootstrap_nodes( + [self.env.d_env.get_node(name='slave-09')]) + self.fuel_web.update_nodes(cluster_id, + {'slave-09': ['standalone-database']}) + self.show_step(3) + self.fuel_web.verify_network(cluster_id) + self.show_step(4) + # LP 1562736 get_devops_node_by_nailgun_node is not working + self.fuel_web.deploy_cluster_wait(cluster_id) + self.show_step(5) + self.fuel_web.run_ostf(cluster_id)