788 lines
30 KiB
Python
788 lines
30 KiB
Python
# Copyright 2015 Mirantis, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
from cinderclient.exceptions import NotFound
|
|
from devops.helpers import helpers as devops_helpers
|
|
from proboscis.asserts import assert_equal
|
|
from proboscis.asserts import assert_true
|
|
from proboscis import test
|
|
|
|
from fuelweb_test import logger
|
|
from fuelweb_test.helpers.decorators import log_snapshot_after_test
|
|
from fuelweb_test.helpers import os_actions
|
|
from fuelweb_test.helpers.ssh_manager import SSHManager
|
|
from fuelweb_test.helpers.utils import preserve_partition
|
|
from fuelweb_test.settings import DEPLOYMENT_MODE
|
|
from fuelweb_test.tests.base_test_case import SetupEnvironment
|
|
from fuelweb_test.tests.base_test_case import TestBasic
|
|
|
|
|
|
@test
|
|
class NodeReinstallationEnv(TestBasic):
|
|
"""NodeReinstallationEnv.""" # TODO documentation
|
|
|
|
@staticmethod
|
|
def reinstall_nodes(fuel_web_client, cluster_id, nodes=None):
|
|
"""Provision and deploy the given cluster nodes."""
|
|
task = fuel_web_client.client.provision_nodes(cluster_id, nodes)
|
|
fuel_web_client.assert_task_success(task)
|
|
task = fuel_web_client.client.deploy_nodes(cluster_id, nodes)
|
|
fuel_web_client.assert_task_success(task)
|
|
|
|
@test(depends_on=[SetupEnvironment.prepare_slaves_5],
|
|
groups=["node_reinstallation_env"])
|
|
@log_snapshot_after_test
|
|
def node_reinstallation_env(self):
|
|
"""Deploy a cluster for nodes reinstallation.
|
|
|
|
Scenario:
|
|
1. Create a cluster
|
|
2. Add 3 nodes with controller and mongo roles
|
|
3. Add a node with compute and cinder roles
|
|
4. Deploy the cluster
|
|
5. Verify that the deployment is completed successfully
|
|
|
|
Duration 190m
|
|
"""
|
|
self.check_run("node_reinstallation_env")
|
|
self.env.revert_snapshot("ready_with_5_slaves")
|
|
|
|
cluster_id = self.fuel_web.create_cluster(
|
|
name=self.__class__.__name__,
|
|
mode=DEPLOYMENT_MODE,
|
|
settings={
|
|
'ceilometer': True,
|
|
}
|
|
)
|
|
|
|
self.fuel_web.update_nodes(
|
|
cluster_id, {
|
|
'slave-01': ['controller', 'mongo'],
|
|
'slave-02': ['controller', 'mongo'],
|
|
'slave-03': ['controller', 'mongo'],
|
|
'slave-04': ['compute', 'cinder'],
|
|
}
|
|
)
|
|
|
|
self.fuel_web.deploy_cluster_wait(cluster_id, check_services=False)
|
|
self.fuel_web.verify_network(cluster_id)
|
|
self.fuel_web.run_ostf(cluster_id)
|
|
|
|
self.env.make_snapshot("node_reinstallation_env", is_make=True)
|
|
|
|
@test(depends_on=[SetupEnvironment.prepare_slaves_5],
|
|
groups=["failed_node_reinstallation_env"])
|
|
@log_snapshot_after_test
|
|
def failed_node_reinstallation_env(self):
|
|
"""Prepare a cluster for 'failed node reinstallation' tests.
|
|
|
|
Scenario:
|
|
1. Revert the snapshot
|
|
2. Create a cluster
|
|
3. Add 3 nodes with controller and mongo roles
|
|
4. Add a node with compute and cinder roles
|
|
5. Provision nodes
|
|
|
|
Duration 25m
|
|
"""
|
|
self.check_run("failed_node_reinstallation_env")
|
|
self.env.revert_snapshot("ready_with_5_slaves")
|
|
|
|
cluster_id = self.fuel_web.create_cluster(
|
|
name=self.__class__.__name__,
|
|
mode=DEPLOYMENT_MODE,
|
|
settings={
|
|
'ceilometer': True,
|
|
}
|
|
)
|
|
|
|
self.fuel_web.update_nodes(
|
|
cluster_id, {
|
|
'slave-01': ['controller', 'mongo'],
|
|
'slave-02': ['controller', 'mongo'],
|
|
'slave-03': ['controller', 'mongo'],
|
|
'slave-04': ['compute', 'cinder'],
|
|
}
|
|
)
|
|
|
|
# Provision nodes
|
|
task = self.fuel_web.client.provision_nodes(cluster_id)
|
|
self.fuel_web.assert_task_success(task)
|
|
|
|
self.env.make_snapshot("failed_node_reinstallation_env", is_make=True)
|
|
|
|
|
|
@test(groups=["ready_node_reinstallation"])
|
|
class ReadyNodeReinstallation(TestBasic):
|
|
"""ReadyNodeReinstallation.""" # TODO documentation
|
|
|
|
@staticmethod
|
|
def _check_hostname(old_node_nailgun, reinstalled_node_nailgun):
|
|
"""Check that the hostname is the same on both given nodes."""
|
|
assert_equal(old_node_nailgun['hostname'],
|
|
reinstalled_node_nailgun['hostname'],
|
|
"Hostname of the reinstalled controller {0} has been "
|
|
"automatically changed to {1} one". format(
|
|
reinstalled_node_nailgun['hostname'],
|
|
old_node_nailgun['hostname']))
|
|
|
|
@test(depends_on=[NodeReinstallationEnv.node_reinstallation_env],
|
|
groups=["reinstall_single_regular_controller_node"])
|
|
@log_snapshot_after_test
|
|
def reinstall_single_regular_controller_node(self):
|
|
"""Verify reinstallation of a regular (non-primary) controller node.
|
|
|
|
Scenario:
|
|
1. Revert snapshot
|
|
2. Select a non-primary controller
|
|
3. Reinstall the controller
|
|
4. Run network verification
|
|
5. Run OSTF
|
|
6. Verify that the hostname is not changed on reinstallation
|
|
of the node
|
|
|
|
Duration: 100m
|
|
"""
|
|
self.env.revert_snapshot("node_reinstallation_env")
|
|
|
|
cluster_id = self.fuel_web.get_last_created_cluster()
|
|
|
|
# Select a non-primary controller
|
|
regular_ctrl = self.fuel_web.get_nailgun_node_by_name("slave-02")
|
|
|
|
# Reinstall the controller
|
|
NodeReinstallationEnv.reinstall_nodes(
|
|
self.fuel_web, cluster_id, [str(regular_ctrl['id'])])
|
|
|
|
self.fuel_web.verify_network(cluster_id)
|
|
self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke', 'sanity'])
|
|
|
|
# Verify that the hostname isn't changed on reinstallation of the node
|
|
self._check_hostname(
|
|
regular_ctrl, self.fuel_web.get_nailgun_node_by_name("slave-02"))
|
|
|
|
@test(depends_on=[NodeReinstallationEnv.node_reinstallation_env],
|
|
groups=["reinstall_single_primary_controller_node"])
|
|
@log_snapshot_after_test
|
|
def reinstall_single_primary_controller_node(self):
|
|
"""Verify reinstallation of the primary controller node.
|
|
|
|
Scenario:
|
|
1. Revert snapshot
|
|
2. Select the primary controller
|
|
3. Reinstall the controller
|
|
4. Run network verification
|
|
5. Run OSTF
|
|
6. Verify that the hostname is not changed on reinstallation
|
|
of the node
|
|
7. Verify that the primary-controller role is not migrated on
|
|
reinstallation of the node
|
|
|
|
Duration: 100m
|
|
"""
|
|
self.env.revert_snapshot("node_reinstallation_env")
|
|
|
|
cluster_id = self.fuel_web.get_last_created_cluster()
|
|
|
|
# Select the primary controller
|
|
primary_ctrl_devops = self.fuel_web.get_nailgun_primary_node(
|
|
self.env.d_env.nodes().slaves[0])
|
|
primary_ctrl_nailgun = self.fuel_web.get_nailgun_node_by_devops_node(
|
|
primary_ctrl_devops)
|
|
|
|
# Reinstall the controller
|
|
NodeReinstallationEnv.reinstall_nodes(
|
|
self.fuel_web, cluster_id, [str(primary_ctrl_nailgun['id'])])
|
|
|
|
self.fuel_web.verify_network(cluster_id)
|
|
self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke', 'sanity'])
|
|
|
|
# Verify that the hostname isn't changed on reinstallation of the node
|
|
self._check_hostname(
|
|
primary_ctrl_nailgun,
|
|
self.fuel_web.get_nailgun_node_by_devops_node(
|
|
primary_ctrl_devops))
|
|
|
|
# Verify that the primary-controller role is not migrated on
|
|
# reinstallation of the node
|
|
reinstalled_primary_ctrl = self.fuel_web.get_nailgun_primary_node(
|
|
self.env.d_env.nodes().slaves[0])
|
|
assert_equal(
|
|
reinstalled_primary_ctrl.name,
|
|
primary_ctrl_devops.name,
|
|
"The primary-controller was migrated from {0} slave to {1} "
|
|
"one.".format(primary_ctrl_devops.name,
|
|
reinstalled_primary_ctrl.name)
|
|
)
|
|
|
|
@test(depends_on=[NodeReinstallationEnv.node_reinstallation_env],
|
|
groups=["reinstall_single_compute_node"])
|
|
@log_snapshot_after_test
|
|
def reinstall_single_compute_node(self):
|
|
"""Verify reinstallation of a compute node.
|
|
|
|
Scenario:
|
|
1. Revert snapshot
|
|
2. Select a compute node
|
|
3. Reinstall the compute
|
|
4. Run network verification
|
|
5. Run OSTF
|
|
6. Verify that the hostname is not changed on reinstallation
|
|
of the node
|
|
|
|
Duration: 55m
|
|
"""
|
|
self.env.revert_snapshot("node_reinstallation_env")
|
|
|
|
cluster_id = self.fuel_web.get_last_created_cluster()
|
|
|
|
# Select a compute
|
|
cmp_nailgun = self.fuel_web.get_nailgun_node_by_name('slave-04')
|
|
|
|
# Reinstall the compute
|
|
logger.info('Reinstall')
|
|
NodeReinstallationEnv.reinstall_nodes(
|
|
self.fuel_web, cluster_id, [str(cmp_nailgun['id'])])
|
|
|
|
logger.info('Verify network')
|
|
self.fuel_web.verify_network(cluster_id)
|
|
self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke', 'sanity'])
|
|
|
|
# Verify that the hostname isn't changed on reinstallation of the node
|
|
self._check_hostname(
|
|
cmp_nailgun, self.fuel_web.get_nailgun_node_by_name('slave-04'))
|
|
|
|
|
|
@test(groups=["full_cluster_reinstallation"])
|
|
class FullClusterReinstallation(TestBasic):
|
|
@test(depends_on=[NodeReinstallationEnv.node_reinstallation_env],
|
|
groups=["full_cluster_reinstallation"])
|
|
@log_snapshot_after_test
|
|
def full_cluster_reinstallation(self):
|
|
"""Verify full cluster reinstallation.
|
|
|
|
Scenario:
|
|
1. Revert snapshot
|
|
2. Create an empty sample file on each node to check that it is not
|
|
available after cluster reinstallation
|
|
3. Reinstall all cluster nodes
|
|
4. Verify that all nodes are reinstalled (not just rebooted),
|
|
i.e. there is no sample file on a node
|
|
5. Run network verification
|
|
6. Run OSTF
|
|
|
|
Duration: 145m
|
|
"""
|
|
self.env.revert_snapshot("node_reinstallation_env")
|
|
|
|
cluster_id = self.fuel_web.get_last_created_cluster()
|
|
|
|
# Create a sample file on each node to check that it is not
|
|
# available after nodes' reinstallation
|
|
file_name = "node_reinstallation.test"
|
|
for slave in self.env.d_env.nodes().slaves[0:4]:
|
|
with self.fuel_web.get_ssh_for_node(slave.name) as remote:
|
|
remote.execute("touch {0}".format(file_name))
|
|
node = self.fuel_web.get_nailgun_node_by_name(slave.name)
|
|
NodeReinstallationEnv.reinstall_nodes(
|
|
self.fuel_web, cluster_id, [str(node['id'])])
|
|
|
|
# Verify that all node are reinstalled (not just rebooted),
|
|
# i.e. there is no sample file on a node
|
|
for slave in self.env.d_env.nodes().slaves[0:4]:
|
|
with self.fuel_web.get_ssh_for_node(slave.name) as remote:
|
|
res = remote.execute("test -e {0}".format(file_name))
|
|
assert_equal(1, res['exit_code'],
|
|
"{0} node was not reinstalled.".format(slave.name))
|
|
|
|
self.fuel_web.verify_network(cluster_id)
|
|
self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke', 'sanity'])
|
|
|
|
|
|
@test(groups=["error_node_reinstallation"])
|
|
class ErrorNodeReinstallation(TestBasic):
|
|
"""ErrorNodeReinstallation.""" # TODO documentation
|
|
|
|
@staticmethod
|
|
def _turnoff_executable_ruby(node):
|
|
"""Set mode -x for /usr/bin/ruby
|
|
|
|
:param node: dict, node attributes
|
|
"""
|
|
ssh = SSHManager()
|
|
cmd = 'chmod -x /usr/bin/ruby'
|
|
ssh.execute_on_remote(node['ip'], cmd)
|
|
|
|
@staticmethod
|
|
def _turnon_executable_ruby(node):
|
|
"""Set mode +x for /usr/bin/ruby
|
|
|
|
:param node: dict, node attributes
|
|
"""
|
|
ssh = SSHManager()
|
|
cmd = 'chmod +x /usr/bin/ruby'
|
|
ssh.execute_on_remote(node['ip'], cmd)
|
|
|
|
def _put_cluster_in_error_state(self, cluster_id, node):
|
|
"""Put cluster in error state
|
|
|
|
:param cluster_id: int, number of cluster id
|
|
:param node: dict, node attributes
|
|
:return:
|
|
"""
|
|
|
|
# Start deployment for corresponding node
|
|
task = self.fuel_web.client.deploy_nodes(
|
|
cluster_id,
|
|
[str(node['id'])])
|
|
# disable ruby and wait for cluster will be in error state
|
|
self._turnoff_executable_ruby(node)
|
|
self.fuel_web.assert_task_failed(task)
|
|
# enable ruby
|
|
self._turnon_executable_ruby(node)
|
|
|
|
@test(depends_on=[NodeReinstallationEnv.failed_node_reinstallation_env],
|
|
groups=["reinstall_failed_primary_controller_deployment"])
|
|
@log_snapshot_after_test
|
|
def reinstall_failed_primary_controller_deployment(self):
|
|
"""Verify reinstallation of a failed controller.
|
|
|
|
Scenario:
|
|
1. Revert the snapshot
|
|
2. Start deployment; fail deployment on primary controller
|
|
3. Reinstall the cluster
|
|
4. Run network verification
|
|
5. Run OSTF
|
|
|
|
Duration: 145m
|
|
"""
|
|
self.show_step(1)
|
|
self.env.revert_snapshot("failed_node_reinstallation_env")
|
|
|
|
self.show_step(2)
|
|
cluster_id = self.fuel_web.get_last_created_cluster()
|
|
# Get the primary controller
|
|
pr_controller = self.fuel_web.get_nailgun_node_by_name('slave-01')
|
|
self._put_cluster_in_error_state(cluster_id, pr_controller)
|
|
|
|
self.show_step(3)
|
|
NodeReinstallationEnv.reinstall_nodes(self.fuel_web, cluster_id)
|
|
|
|
self.show_step(4)
|
|
self.fuel_web.verify_network(cluster_id)
|
|
|
|
self.show_step(5)
|
|
self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke', 'sanity'])
|
|
|
|
@test(depends_on=[NodeReinstallationEnv.failed_node_reinstallation_env],
|
|
groups=["reinstall_failed_regular_controller_deployment"])
|
|
@log_snapshot_after_test
|
|
def reinstall_failed_regular_controller_deployment(self):
|
|
"""Verify reinstallation of a failed controller.
|
|
|
|
Scenario:
|
|
1. Revert the snapshot
|
|
2. Start deployment; fail deployment on regular controller
|
|
3. Reinstall the cluster
|
|
4. Run network verification
|
|
5. Run OSTF
|
|
|
|
Duration: 145m
|
|
"""
|
|
self.show_step(1)
|
|
self.env.revert_snapshot("failed_node_reinstallation_env")
|
|
|
|
self.show_step(2)
|
|
cluster_id = self.fuel_web.get_last_created_cluster()
|
|
# Get a regular controller
|
|
regular_ctrl = self.fuel_web.get_nailgun_node_by_name('slave-02')
|
|
self._put_cluster_in_error_state(cluster_id, regular_ctrl)
|
|
|
|
self.show_step(3)
|
|
NodeReinstallationEnv.reinstall_nodes(self.fuel_web, cluster_id)
|
|
|
|
self.show_step(4)
|
|
self.fuel_web.verify_network(cluster_id)
|
|
|
|
self.show_step(5)
|
|
self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke', 'sanity'])
|
|
|
|
@test(depends_on=[NodeReinstallationEnv.failed_node_reinstallation_env],
|
|
groups=["reinstall_failed_compute_deployment"])
|
|
@log_snapshot_after_test
|
|
def reinstall_failed_compute_deployment(self):
|
|
"""Verify reinstallation of a failed compute.
|
|
|
|
Scenario:
|
|
1. Revert the snapshot
|
|
2. Start deployment; fail deployment on one of computes
|
|
3. Reinstall the cluster
|
|
4. Run network verification
|
|
5. Run OSTF
|
|
|
|
Duration: 45m
|
|
"""
|
|
self.show_step(1)
|
|
self.env.revert_snapshot("failed_node_reinstallation_env")
|
|
|
|
self.show_step(2)
|
|
cluster_id = self.fuel_web.get_last_created_cluster()
|
|
# Get nailgun nodes
|
|
nailgun_nodes = self.fuel_web.client.list_cluster_nodes(cluster_id)
|
|
cmps_nailgun = [n for n in nailgun_nodes
|
|
if 'compute' in n['pending_roles']]
|
|
self._put_cluster_in_error_state(cluster_id, cmps_nailgun[0])
|
|
|
|
self.show_step(3)
|
|
NodeReinstallationEnv.reinstall_nodes(self.fuel_web, cluster_id)
|
|
|
|
self.show_step(4)
|
|
self.fuel_web.verify_network(cluster_id)
|
|
|
|
self.show_step(5)
|
|
self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke', 'sanity'])
|
|
|
|
|
|
@test(groups=["partition_preservation"],
|
|
enabled=False)
|
|
class PartitionPreservation(TestBasic):
|
|
"""PartitionPreservation.""" # TODO documentation
|
|
|
|
@test(depends_on=[NodeReinstallationEnv.node_reinstallation_env],
|
|
groups=["cinder_nova_partition_preservation"])
|
|
@log_snapshot_after_test
|
|
def cinder_nova_partition_preservation(self):
|
|
"""Verify partition preservation of Cinder and Nova instances data.
|
|
|
|
Scenario:
|
|
1. Revert the snapshot
|
|
2. Create an OS volume and OS instance
|
|
3. Mark 'cinder' partition to be preserved
|
|
4. Mark 'vm' partition to be preserved
|
|
5. Reinstall the compute node
|
|
6. Run network verification
|
|
7. Run OSTF
|
|
8. Verify that the volume is present and has 'available' status
|
|
after the node reinstallation
|
|
9. Verify that the VM is available and pingable
|
|
after the node reinstallation
|
|
|
|
Duration: 115m
|
|
|
|
"""
|
|
self.env.revert_snapshot("node_reinstallation_env")
|
|
|
|
cluster_id = self.fuel_web.get_last_created_cluster()
|
|
|
|
# Create an OS volume
|
|
os_conn = os_actions.OpenStackActions(
|
|
self.fuel_web.get_public_vip(cluster_id))
|
|
|
|
volume = os_conn.create_volume()
|
|
|
|
# Create an OS instance
|
|
cmp_host = os_conn.get_hypervisors()[0]
|
|
|
|
net_label = self.fuel_web.get_cluster_predefined_networks_name(
|
|
cluster_id)['private_net']
|
|
|
|
vm = os_conn.create_server_for_migration(
|
|
neutron=True,
|
|
availability_zone="nova:{0}".format(
|
|
cmp_host.hypervisor_hostname), label=net_label)
|
|
vm_floating_ip = os_conn.assign_floating_ip(vm)
|
|
devops_helpers.wait(
|
|
lambda: devops_helpers.tcp_ping(vm_floating_ip.ip, 22),
|
|
timeout=120)
|
|
|
|
cmp_nailgun = self.fuel_web.get_nailgun_node_by_fqdn(
|
|
cmp_host.hypervisor_hostname)
|
|
|
|
# Mark 'cinder' and 'vm' partitions to be preserved
|
|
with self.env.d_env.get_admin_remote() as remote:
|
|
preserve_partition(remote, cmp_nailgun['id'], "cinder")
|
|
preserve_partition(remote, cmp_nailgun['id'], "vm")
|
|
|
|
NodeReinstallationEnv.reinstall_nodes(
|
|
self.fuel_web, cluster_id, [str(cmp_nailgun['id'])])
|
|
|
|
self.fuel_web.verify_network(cluster_id)
|
|
self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke', 'sanity'])
|
|
|
|
# Verify that the created volume is still available
|
|
try:
|
|
volume = os_conn.cinder.volumes.get(volume.id)
|
|
except NotFound:
|
|
raise AssertionError(
|
|
"{0} volume is not available after its {1} hosting node "
|
|
"reinstallation".format(volume.id, cmp_nailgun['fqdn']))
|
|
expected_status = "available"
|
|
assert_equal(
|
|
expected_status,
|
|
volume.status,
|
|
"{0} volume status is {1} after its {2} hosting node "
|
|
"reinstallation. Expected status is {3}.".format(
|
|
volume.id, volume.status, cmp_nailgun['fqdn'], expected_status)
|
|
)
|
|
|
|
# Verify that the VM is still available
|
|
try:
|
|
os_conn.verify_instance_status(vm, 'ACTIVE')
|
|
except AssertionError:
|
|
raise AssertionError(
|
|
"{0} VM is not available after its {1} hosting node "
|
|
"reinstallation".format(vm.name,
|
|
cmp_host.hypervisor_hostname))
|
|
assert_true(devops_helpers.tcp_ping(vm_floating_ip.ip, 22),
|
|
"{0} VM is not accessible via its {1} floating "
|
|
"ip".format(vm.name, vm_floating_ip))
|
|
|
|
@test(depends_on=[NodeReinstallationEnv.node_reinstallation_env],
|
|
groups=["mongo_mysql_partition_preservation"])
|
|
@log_snapshot_after_test
|
|
def mongo_mysql_partition_preservation(self):
|
|
"""Verify partition preservation of Ceilometer and mysql data.
|
|
|
|
Scenario:
|
|
1. Revert the snapshot
|
|
2. Create a ceilometer alarm
|
|
3. Mark 'mongo' and 'mysql' partitions to be
|
|
preserved on one of controllers
|
|
4. Reinstall the controller
|
|
5. Verify that the alarm is present after the node reinstallation
|
|
6. Verify that the reinstalled node joined the Galera cluster
|
|
and synced its state
|
|
7. Run network verification
|
|
8. Run OSTF
|
|
|
|
Duration: 110m
|
|
|
|
"""
|
|
self.env.revert_snapshot("node_reinstallation_env")
|
|
|
|
cluster_id = self.fuel_web.get_last_created_cluster()
|
|
|
|
# Create a ceilometer alarm
|
|
with self.fuel_web.get_ssh_for_node("slave-01") as remote:
|
|
alarm_name = "test_alarm"
|
|
res = remote.execute(
|
|
"source openrc; "
|
|
"ceilometer alarm-threshold-create "
|
|
"--name {0} "
|
|
"-m {1} "
|
|
"--threshold {2}".format(alarm_name, "cpu_util", "80.0")
|
|
)
|
|
assert_equal(0, res['exit_code'],
|
|
"Creating alarm via ceilometer CLI failed.")
|
|
initial_alarms = remote.execute(
|
|
"source openrc; ceilometer alarm-list")
|
|
|
|
mongo_nailgun = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
|
|
cluster_id, ['mongo'])[0]
|
|
|
|
# Mark 'mongo' and 'mysql' partitions to be preserved
|
|
with self.env.d_env.get_admin_remote() as remote:
|
|
preserve_partition(remote, mongo_nailgun['id'], "mongo")
|
|
preserve_partition(remote, mongo_nailgun['id'], "mysql")
|
|
|
|
NodeReinstallationEnv.reinstall_nodes(
|
|
self.fuel_web, cluster_id, [str(mongo_nailgun['id'])])
|
|
|
|
with self.fuel_web.get_ssh_for_nailgun_node(mongo_nailgun) as rmt:
|
|
alarms = rmt.execute("source openrc; ceilometer alarm-list")
|
|
assert_equal(
|
|
initial_alarms['stdout'],
|
|
alarms['stdout'],
|
|
"{0} alarm is not available in mongo after reinstallation "
|
|
"of the controllers".format(alarm_name))
|
|
|
|
cmd = ("mysql --connect_timeout=5 -sse "
|
|
"\"SHOW STATUS LIKE 'wsrep%';\"")
|
|
err_msg = ("Galera isn't ready on {0} "
|
|
"node".format(mongo_nailgun['hostname']))
|
|
devops_helpers.wait(
|
|
lambda: rmt.execute(cmd)['exit_code'] == 0,
|
|
timeout=10 * 60,
|
|
timeout_msg=err_msg)
|
|
|
|
cmd = ("mysql --connect_timeout=5 -sse \"SHOW STATUS LIKE "
|
|
"'wsrep_local_state_comment';\"")
|
|
err_msg = ("The reinstalled node {0} is not synced with the "
|
|
"Galera cluster".format(mongo_nailgun['hostname']))
|
|
devops_helpers.wait(
|
|
lambda: rmt.execute(cmd)['stdout'][0].split()[1] == "Synced",
|
|
timeout=10 * 60,
|
|
timeout_msg=err_msg)
|
|
|
|
self.fuel_web.verify_network(cluster_id)
|
|
self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke', 'sanity'])
|
|
|
|
|
|
@test(groups=["known_issues"])
|
|
class StopReinstallation(TestBasic):
|
|
"""StopReinstallation.""" # TODO documentation
|
|
|
|
@staticmethod
|
|
def _stop_reinstallation(fuel_web_client, cluster_id, node, slave_nodes):
|
|
|
|
logger.info('Start reinstall')
|
|
|
|
task = fuel_web_client.client.provision_nodes(cluster_id, node)
|
|
fuel_web_client.assert_task_success(task)
|
|
task = fuel_web_client.client.deploy_nodes(cluster_id, node)
|
|
fuel_web_client.assert_task_success(task, progress=60)
|
|
|
|
logger.info('Stop reinstall')
|
|
fuel_web_client.stop_deployment_wait(cluster_id)
|
|
fuel_web_client.wait_nodes_get_online_state(
|
|
slave_nodes,
|
|
timeout=8 * 60)
|
|
|
|
@test(depends_on=[NodeReinstallationEnv.node_reinstallation_env],
|
|
groups=["compute_stop_reinstallation"])
|
|
@log_snapshot_after_test
|
|
def compute_stop_reinstallation(self):
|
|
"""Verify stop reinstallation of compute.
|
|
|
|
Scenario:
|
|
1. Revert the snapshot
|
|
2. Create an OS volume and OS instance
|
|
3. Mark 'cinder' and 'vm' partitions to be preserved
|
|
4. Stop reinstallation process of compute
|
|
5. Start the reinstallation process again
|
|
6. Run network verification
|
|
7. Run OSTF
|
|
8. Verify that the volume is present and has 'available' status
|
|
after the node reinstallation
|
|
9. Verify that the VM is available and pingable
|
|
after the node reinstallation
|
|
|
|
Duration: 115m
|
|
|
|
"""
|
|
self.env.revert_snapshot("node_reinstallation_env")
|
|
|
|
cluster_id = self.fuel_web.get_last_created_cluster()
|
|
|
|
# Create an OS volume
|
|
os_conn = os_actions.OpenStackActions(
|
|
self.fuel_web.get_public_vip(cluster_id))
|
|
|
|
volume = os_conn.create_volume()
|
|
|
|
# Create an OS instance
|
|
cmp_host = os_conn.get_hypervisors()[0]
|
|
|
|
net_label = self.fuel_web.get_cluster_predefined_networks_name(
|
|
cluster_id)['private_net']
|
|
|
|
vm = os_conn.create_server_for_migration(
|
|
neutron=True,
|
|
availability_zone="nova:{0}".format(
|
|
cmp_host.hypervisor_hostname), label=net_label)
|
|
vm_floating_ip = os_conn.assign_floating_ip(vm)
|
|
devops_helpers.wait(
|
|
lambda: devops_helpers.tcp_ping(vm_floating_ip.ip, 22),
|
|
timeout=120)
|
|
|
|
cmp_nailgun = self.fuel_web.get_nailgun_node_by_fqdn(
|
|
cmp_host.hypervisor_hostname)
|
|
|
|
# Mark 'cinder' and 'vm' partitions to be preserved
|
|
with self.env.d_env.get_admin_remote() as remote:
|
|
preserve_partition(remote, cmp_nailgun['id'], "cinder")
|
|
preserve_partition(remote, cmp_nailgun['id'], "vm")
|
|
|
|
slave_nodes = self.fuel_web.client.list_cluster_nodes(cluster_id)
|
|
devops_nodes = self.fuel_web.get_devops_nodes_by_nailgun_nodes(
|
|
slave_nodes)
|
|
|
|
logger.info('Stop reinstallation process')
|
|
self._stop_reinstallation(self.fuel_web, cluster_id,
|
|
[str(cmp_nailgun['id'])], devops_nodes)
|
|
|
|
self.fuel_web.verify_network(cluster_id)
|
|
logger.info('Start the reinstallation process again')
|
|
NodeReinstallationEnv.reinstall_nodes(
|
|
self.fuel_web, cluster_id, [str(cmp_nailgun['id'])])
|
|
|
|
self.fuel_web.verify_network(cluster_id)
|
|
self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke', 'sanity'])
|
|
|
|
# Verify that the created volume is still available
|
|
try:
|
|
volume = os_conn.cinder.volumes.get(volume.id)
|
|
except NotFound:
|
|
raise AssertionError(
|
|
"{0} volume is not available after its {1} hosting node "
|
|
"reinstallation".format(volume.id, cmp_nailgun['fqdn']))
|
|
expected_status = "available"
|
|
assert_equal(
|
|
expected_status,
|
|
volume.status,
|
|
"{0} volume status is {1} after its {2} hosting node "
|
|
"reinstallation. Expected status is {3}.".format(
|
|
volume.id, volume.status, cmp_nailgun['fqdn'], expected_status)
|
|
)
|
|
|
|
# Verify that the VM is still available
|
|
try:
|
|
os_conn.verify_instance_status(vm, 'ACTIVE')
|
|
except AssertionError:
|
|
raise AssertionError(
|
|
"{0} VM is not available after its {1} hosting node "
|
|
"reinstallation".format(vm.name,
|
|
cmp_host.hypervisor_hostname))
|
|
assert_true(devops_helpers.tcp_ping(vm_floating_ip.ip, 22),
|
|
"{0} VM is not accessible via its {1} floating "
|
|
"ip".format(vm.name, vm_floating_ip))
|
|
|
|
@test(depends_on=[NodeReinstallationEnv.node_reinstallation_env],
|
|
groups=["node_stop_reinstallation"])
|
|
@log_snapshot_after_test
|
|
def node_stop_reinstallation(self):
|
|
"""Verify stop reinstallation of node.
|
|
|
|
Scenario:
|
|
1. Revert the snapshot
|
|
2. Stop reinstallation process of node
|
|
3. Start the reinstallation process again
|
|
4. Run network verification
|
|
5. Run OSTF
|
|
|
|
Duration: 115m
|
|
|
|
"""
|
|
self.env.revert_snapshot("node_reinstallation_env")
|
|
|
|
cluster_id = self.fuel_web.get_last_created_cluster()
|
|
|
|
# Select a node
|
|
ctrl_nailgun = self.fuel_web.get_nailgun_node_by_name('slave-01')
|
|
|
|
slave_nodes = self.fuel_web.client.list_cluster_nodes(cluster_id)
|
|
devops_nodes = self.fuel_web.get_devops_nodes_by_nailgun_nodes(
|
|
slave_nodes)
|
|
|
|
logger.info('Stop reinstallation process of node')
|
|
self._stop_reinstallation(self.fuel_web, cluster_id,
|
|
[str(ctrl_nailgun['id'])], devops_nodes)
|
|
|
|
logger.info('Start the reinstallation process again')
|
|
NodeReinstallationEnv.reinstall_nodes(
|
|
self.fuel_web, cluster_id, [str(ctrl_nailgun['id'])])
|
|
|
|
self.fuel_web.verify_network(cluster_id)
|
|
self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke', 'sanity'])
|