ffef5be826
SSHClient instance is already has this method
Blueprint: sshmanager-integration
Change-Id: I90ca8bc7675bfc6b70126f83fd4e6ec0e3e04d3a
(cherry picked from commit d37e14d)
944 lines
35 KiB
Python
944 lines
35 KiB
Python
# Copyright 2014 Mirantis, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
from __future__ import unicode_literals
|
|
|
|
import time
|
|
|
|
import paramiko
|
|
from pkg_resources import parse_version
|
|
from proboscis.asserts import assert_true, assert_false, assert_equal
|
|
from proboscis import SkipTest
|
|
from proboscis import test
|
|
from devops.helpers.helpers import tcp_ping
|
|
from devops.helpers.helpers import wait
|
|
from devops.helpers.ssh_client import SSHAuth
|
|
from six import BytesIO
|
|
|
|
from fuelweb_test.helpers import os_actions
|
|
from fuelweb_test.helpers import ceph
|
|
from fuelweb_test.helpers import checkers
|
|
from fuelweb_test.helpers import utils
|
|
from fuelweb_test.helpers.decorators import log_snapshot_after_test
|
|
from fuelweb_test.helpers.ovs import ovs_get_tag_by_port
|
|
from fuelweb_test import ostf_test_mapping
|
|
from fuelweb_test import settings
|
|
from fuelweb_test import logger
|
|
from fuelweb_test.tests.base_test_case import SetupEnvironment
|
|
from fuelweb_test.tests.base_test_case import TestBasic
|
|
|
|
|
|
@test(groups=["ceph_ha_one_controller", "ceph"])
|
|
class CephCompact(TestBasic):
|
|
"""CephCompact.""" # TODO documentation
|
|
|
|
@test(depends_on=[SetupEnvironment.prepare_slaves_3],
|
|
groups=["ceph_ha_one_controller_compact",
|
|
"ha_one_controller_nova_ceph",
|
|
"ceph_ha_one_controller_compact_neutron", "ceph",
|
|
"nova", "deployment"])
|
|
@log_snapshot_after_test
|
|
def ceph_ha_one_controller_compact(self):
|
|
"""Deploy ceph in HA mode with 1 controller
|
|
|
|
Scenario:
|
|
1. Create cluster
|
|
2. Add 1 node with controller and ceph OSD roles
|
|
3. Add 2 nodes with compute and ceph OSD roles
|
|
4. Deploy the cluster
|
|
5. Check ceph status
|
|
|
|
Duration 35m
|
|
Snapshot ceph_ha_one_controller_compact
|
|
"""
|
|
self.check_run('ceph_ha_one_controller_compact')
|
|
self.env.revert_snapshot("ready_with_3_slaves")
|
|
data = {
|
|
'volumes_ceph': True,
|
|
'images_ceph': True,
|
|
'volumes_lvm': False,
|
|
'tenant': 'ceph1',
|
|
'user': 'ceph1',
|
|
'password': 'ceph1',
|
|
'net_provider': 'neutron',
|
|
'net_segment_type': settings.NEUTRON_SEGMENT['vlan']
|
|
}
|
|
|
|
cluster_id = self.fuel_web.create_cluster(
|
|
name=self.__class__.__name__,
|
|
mode=settings.DEPLOYMENT_MODE,
|
|
settings=data)
|
|
self.fuel_web.update_nodes(
|
|
cluster_id,
|
|
{
|
|
'slave-01': ['controller', 'ceph-osd'],
|
|
'slave-02': ['compute', 'ceph-osd'],
|
|
'slave-03': ['compute', 'ceph-osd']
|
|
}
|
|
)
|
|
# Cluster deploy
|
|
self.fuel_web.deploy_cluster_wait(cluster_id)
|
|
self.fuel_web.check_ceph_status(cluster_id)
|
|
|
|
# Run ostf
|
|
self.fuel_web.run_ostf(cluster_id=cluster_id)
|
|
|
|
self.env.make_snapshot("ceph_ha_one_controller_compact", is_make=True)
|
|
|
|
@test(depends_on=[ceph_ha_one_controller_compact],
|
|
groups=["check_ceph_cinder_cow"])
|
|
@log_snapshot_after_test
|
|
def check_ceph_cinder_cow(self):
|
|
"""Check copy-on-write when Cinder creates a volume from Glance image
|
|
|
|
Scenario:
|
|
1. Revert a snapshot where ceph enabled for volumes and images:
|
|
"ceph_ha_one_controller_compact"
|
|
2. Create a Glance image in RAW disk format
|
|
3. Create a Cinder volume using Glance image in RAW disk format
|
|
4. Check on a ceph-osd node if the volume has a parent image.
|
|
|
|
Duration 5m
|
|
"""
|
|
self.env.revert_snapshot("ceph_ha_one_controller_compact")
|
|
cluster_id = self.fuel_web.get_last_created_cluster()
|
|
os_conn = os_actions.OpenStackActions(
|
|
self.fuel_web.get_public_vip(cluster_id), 'ceph1', 'ceph1',
|
|
'ceph1')
|
|
|
|
image_data = BytesIO(
|
|
self.__class__.__name__.encode(encoding='ascii', errors='ignore'))
|
|
image = os_conn.create_image(disk_format='raw',
|
|
container_format='bare',
|
|
name='test_ceph_cinder_cow',
|
|
is_public=True,
|
|
data=image_data)
|
|
wait(lambda: os_conn.get_image(image.name).status == 'active',
|
|
timeout=60 * 2, timeout_msg='Image is not active')
|
|
|
|
volume = os_conn.create_volume(size=1, image_id=image.id)
|
|
|
|
with self.fuel_web.get_ssh_for_node('slave-01') as remote:
|
|
rbd_list = ceph.get_rbd_images_list(remote, 'volumes')
|
|
|
|
for item in rbd_list:
|
|
if volume.id in item['image']:
|
|
assert_true('parent' in item,
|
|
"Volume {0} created from image {1} doesn't have"
|
|
" parents. Copy-on-write feature doesn't work."
|
|
.format(volume.id, image.id))
|
|
assert_true(image.id in item['parent']['image'],
|
|
"Volume {0} created from image {1}, but have a "
|
|
"different image in parent: {2}"
|
|
.format(volume.id, image.id,
|
|
item['parent']['image']))
|
|
break
|
|
else:
|
|
raise Exception("Volume {0} not found!".format(volume.id))
|
|
|
|
|
|
@test(groups=["thread_3", "ceph"])
|
|
class CephCompactWithCinder(TestBasic):
|
|
"""CephCompactWithCinder.""" # TODO documentation
|
|
|
|
@test(depends_on=[SetupEnvironment.prepare_release],
|
|
groups=["ceph_ha_one_controller_with_cinder"])
|
|
@log_snapshot_after_test
|
|
def ceph_ha_one_controller_with_cinder(self):
|
|
"""Deploy ceph with cinder in ha mode with 1 controller
|
|
|
|
Scenario:
|
|
1. Create cluster
|
|
2. Add 1 node with controller role
|
|
3. Add 1 node with compute role
|
|
4. Add 2 nodes with cinder and ceph OSD roles
|
|
5. Deploy the cluster
|
|
6. Check ceph status
|
|
7. Check partitions on controller node
|
|
|
|
Duration 40m
|
|
Snapshot ceph_ha_one_controller_with_cinder
|
|
"""
|
|
try:
|
|
self.check_run('ceph_ha_one_controller_with_cinder')
|
|
except SkipTest:
|
|
return
|
|
|
|
self.env.revert_snapshot("ready")
|
|
self.env.bootstrap_nodes(
|
|
self.env.d_env.nodes().slaves[:4])
|
|
|
|
cluster_id = self.fuel_web.create_cluster(
|
|
name=self.__class__.__name__,
|
|
mode=settings.DEPLOYMENT_MODE,
|
|
settings={
|
|
'volumes_ceph': False,
|
|
'images_ceph': True,
|
|
'osd_pool_size': '2',
|
|
'volumes_lvm': True,
|
|
'tenant': 'ceph2',
|
|
'user': 'ceph2',
|
|
'password': 'ceph2'
|
|
}
|
|
)
|
|
self.fuel_web.update_nodes(
|
|
cluster_id,
|
|
{
|
|
'slave-01': ['controller'],
|
|
'slave-02': ['compute'],
|
|
'slave-03': ['cinder', 'ceph-osd'],
|
|
'slave-04': ['cinder', 'ceph-osd']
|
|
}
|
|
)
|
|
# Cluster deploy
|
|
self.fuel_web.deploy_cluster_wait(cluster_id)
|
|
self.fuel_web.check_ceph_status(cluster_id)
|
|
|
|
disks = self.fuel_web.client.get_node_disks(
|
|
self.fuel_web.get_nailgun_node_by_name('slave-01')['id'])
|
|
|
|
logger.info("Current disk partitions are: \n{d}".format(d=disks))
|
|
|
|
logger.info("Check unallocated space")
|
|
# We expect failure here only for release 5.0 due to bug
|
|
# https://bugs.launchpad.net/fuel/+bug/1306625, so it is
|
|
# necessary to assert_true in the next release.
|
|
assert_false(
|
|
checkers.check_unallocated_space(disks, contr_img_ceph=True),
|
|
"Check unallocated space on controller")
|
|
|
|
# Run ostf
|
|
self.fuel_web.run_ostf(cluster_id=cluster_id)
|
|
|
|
self.env.make_snapshot("ceph_ha_one_controller_with_cinder",
|
|
is_make=True)
|
|
|
|
|
|
@test(groups=["thread_3", "ceph"])
|
|
class CephHA(TestBasic):
|
|
"""CephHA.""" # TODO documentation1
|
|
|
|
@test(depends_on=[SetupEnvironment.prepare_release],
|
|
groups=["ceph_ha", "classic_provisioning"])
|
|
@log_snapshot_after_test
|
|
def ceph_ha(self):
|
|
"""Deploy ceph with cinder in HA mode
|
|
|
|
Scenario:
|
|
1. Create cluster
|
|
2. Add 3 nodes with controller and ceph OSD roles
|
|
3. Add 1 node with ceph OSD roles
|
|
4. Add 2 nodes with compute and ceph OSD roles
|
|
5. Deploy the cluster
|
|
|
|
Duration 90m
|
|
Snapshot ceph_ha
|
|
|
|
"""
|
|
try:
|
|
self.check_run('ceph_ha')
|
|
except SkipTest:
|
|
return
|
|
|
|
self.env.revert_snapshot("ready")
|
|
self.env.bootstrap_nodes(
|
|
self.env.d_env.nodes().slaves[:6])
|
|
|
|
data = {
|
|
'volumes_ceph': True,
|
|
'images_ceph': True,
|
|
'volumes_lvm': False,
|
|
'tenant': 'cephHA',
|
|
'user': 'cephHA',
|
|
'password': 'cephHA',
|
|
'osd_pool_size': "3",
|
|
'net_provider': 'neutron',
|
|
'net_segment_type': settings.NEUTRON_SEGMENT['vlan']
|
|
}
|
|
cluster_id = self.fuel_web.create_cluster(
|
|
name=self.__class__.__name__,
|
|
mode=settings.DEPLOYMENT_MODE,
|
|
settings=data
|
|
)
|
|
self.fuel_web.update_nodes(
|
|
cluster_id,
|
|
{
|
|
'slave-01': ['controller', 'ceph-osd'],
|
|
'slave-02': ['controller', 'ceph-osd'],
|
|
'slave-03': ['controller', 'ceph-osd'],
|
|
'slave-04': ['compute', 'ceph-osd'],
|
|
'slave-05': ['compute', 'ceph-osd'],
|
|
'slave-06': ['ceph-osd']
|
|
}
|
|
)
|
|
# Deploy cluster
|
|
self.fuel_web.deploy_cluster_wait(cluster_id)
|
|
|
|
self.env.make_snapshot("ceph_ha", is_make=True)
|
|
|
|
@test(depends_on=[ceph_ha],
|
|
groups=["ha_nova_ceph", "ha_neutron_ceph", "check_ceph_ha"])
|
|
@log_snapshot_after_test
|
|
def check_ceph_ha(self):
|
|
"""Check ceph with cinder in HA mode
|
|
|
|
Scenario:
|
|
1. Revert snapshot with ceph cluster in HA mode
|
|
2. Check ceph status
|
|
3. Check ceph version, should be consistent across nodes
|
|
|
|
Duration 10m
|
|
Snapshot check_ceph_ha
|
|
|
|
"""
|
|
self.env.revert_snapshot("ceph_ha")
|
|
cluster_id = self.fuel_web.get_last_created_cluster()
|
|
|
|
self.fuel_web.check_ceph_status(cluster_id)
|
|
|
|
versions = []
|
|
for node in self.fuel_web.client.list_cluster_nodes(cluster_id):
|
|
role = '_'.join(node['roles'])
|
|
logger.debug('{} has role {}'.format(node['fqdn'], role))
|
|
with self.fuel_web.get_ssh_for_nailgun_node(node) as remote:
|
|
version = ceph.get_version(remote)
|
|
logger.info('On {} ceph version is {}'.format(node['fqdn'],
|
|
version))
|
|
versions.append({'name': node['fqdn'], 'ceph_version': version})
|
|
|
|
ceph_version = versions[0]['ceph_version']
|
|
|
|
bad_nodes = [
|
|
ver for ver in versions
|
|
if parse_version(ver['ceph_version']) != parse_version(
|
|
ceph_version)]
|
|
|
|
assert_true(len(bad_nodes) == 0,
|
|
message="Nodes should same Ceph version on all nodes. "
|
|
"Expecting version {0}, the following nodes "
|
|
"do not have this version: {1}".format(
|
|
ceph_version, bad_nodes))
|
|
# Run ostf
|
|
self.fuel_web.run_ostf(
|
|
cluster_id=cluster_id,
|
|
test_sets=['ha', 'smoke', 'sanity'])
|
|
|
|
@test(depends_on=[ceph_ha],
|
|
groups=["openstack_stat"])
|
|
@log_snapshot_after_test
|
|
def check_openstack_stat(self):
|
|
"""Check openstack statistic on fuel and collector side
|
|
|
|
Scenario:
|
|
1. Revert ceph_ha env
|
|
2. Create all openstack resources that are collected
|
|
3. Check that all info was collected on fuel side
|
|
4. Check that info was sent to collector
|
|
5. Check that info is properly saved on collector side
|
|
|
|
Duration 20m
|
|
Snapshot check_openstack_stat
|
|
|
|
"""
|
|
self.env.revert_snapshot("ceph_ha")
|
|
cluster_id = self.fuel_web.get_last_created_cluster()
|
|
os_conn = os_actions.OpenStackActions(
|
|
self.fuel_web.get_public_vip(cluster_id), 'cephHA', 'cephHA',
|
|
'cephHA')
|
|
|
|
# Check resources addition
|
|
# create instance
|
|
net_name = self.fuel_web.get_cluster_predefined_networks_name(
|
|
cluster_id)['private_net']
|
|
server = os_conn.create_instance(
|
|
neutron_network=True, label=net_name)
|
|
|
|
# create flavor
|
|
flavor = os_conn.create_flavor('openstackstat', 1024, 1, 1)
|
|
|
|
# create volume
|
|
volume = os_conn.create_volume()
|
|
|
|
# create image
|
|
devops_node = self.fuel_web.get_nailgun_primary_node(
|
|
self.env.d_env.nodes().slaves[0])
|
|
with self.fuel_web.get_ssh_for_node(devops_node.name) as slave:
|
|
if settings.OPENSTACK_RELEASE_CENTOS in settings.OPENSTACK_RELEASE:
|
|
slave.execute(". openrc; glance image-create --name"
|
|
" 'custom-image' --disk-format qcow2"
|
|
" --protected False --visibility public"
|
|
" --container-format bare"
|
|
" --file /opt/vm/cirros-x86_64-disk.img")
|
|
else:
|
|
slave.execute(
|
|
". openrc; glance image-create --name"
|
|
" 'custom-image' --disk-format qcow2"
|
|
" --protected False --visibility public"
|
|
" --container-format bare --file"
|
|
" /usr/share/cirros-testvm/cirros-x86_64-disk.img")
|
|
|
|
image = os_conn.get_image_by_name('custom-image')
|
|
logger.debug("image is {}".format(image))
|
|
|
|
# create tenant and user
|
|
tenant = os_conn.create_tenant("openstack_tenant")
|
|
user = os_conn.create_user('openstack_user', 'qwerty', tenant)
|
|
|
|
self.env.nailgun_actions.force_oswl_collect()
|
|
self.env.nailgun_actions.force_fuel_stats_sending()
|
|
master_uid = self.env.get_masternode_uuid()
|
|
checkers.check_oswl_stat(self.env.postgres_actions,
|
|
self.env.nailgun_actions, self.env.collector,
|
|
master_uid, operation='current',
|
|
resources=['vm', 'flavor', 'volume', 'image',
|
|
'tenant', 'keystone_user'])
|
|
|
|
# Check resources modification
|
|
# suspend instance
|
|
server.suspend()
|
|
# edit volume
|
|
os_conn.extend_volume(volume, 2)
|
|
# edit image
|
|
os_conn.update_image(image, min_ram=333)
|
|
# edit user
|
|
os_conn.update_user_enabled(user, enabled=False)
|
|
# edit tenant
|
|
os_conn.update_tenant(tenant.id, enabled=False)
|
|
|
|
self.env.nailgun_actions.force_oswl_collect()
|
|
self.env.nailgun_actions.force_fuel_stats_sending()
|
|
checkers.check_oswl_stat(self.env.postgres_actions,
|
|
self.env.nailgun_actions, self.env.collector,
|
|
master_uid, operation='modified',
|
|
resources=['vm', 'volume', 'image',
|
|
'tenant', 'keystone_user'])
|
|
|
|
# Check resources deletion
|
|
# delete instance
|
|
server.delete()
|
|
# delete flavor
|
|
os_conn.delete_flavor(flavor)
|
|
# delete volume
|
|
os_conn.delete_volume_and_wait(volume, timeout=300)
|
|
# delete image
|
|
os_conn.delete_image(image.id)
|
|
# delete tenant
|
|
os_conn.delete_tenant(tenant)
|
|
# delete user
|
|
os_conn.delete_user(user)
|
|
|
|
self.env.nailgun_actions.force_oswl_collect()
|
|
self.env.nailgun_actions.force_fuel_stats_sending()
|
|
checkers.check_oswl_stat(self.env.postgres_actions,
|
|
self.env.nailgun_actions, self.env.collector,
|
|
master_uid, operation='removed',
|
|
resources=['vm', 'flavor', 'volume', 'image',
|
|
'tenant', 'keystone_user'])
|
|
|
|
|
|
@test(groups=["ha_neutron_tun", "ceph"])
|
|
class CephRadosGW(TestBasic):
|
|
"""CephRadosGW.""" # TODO documentation
|
|
|
|
@test(depends_on=[SetupEnvironment.prepare_release],
|
|
groups=["ceph_rados_gw", "bvt_2", "ceph", "neutron", "deployment"])
|
|
@log_snapshot_after_test
|
|
def ceph_rados_gw(self):
|
|
"""Deploy ceph HA with RadosGW for objects
|
|
|
|
Scenario:
|
|
1. Create cluster with Neutron
|
|
2. Add 3 nodes with controller role
|
|
3. Add 3 nodes with compute and ceph-osd role
|
|
4. Deploy the cluster
|
|
5. Check ceph status
|
|
6. Run OSTF tests
|
|
7. Check the radosgw daemon is started
|
|
|
|
Duration 90m
|
|
Snapshot ceph_rados_gw
|
|
|
|
"""
|
|
def radosgw_started(remote):
|
|
return remote.check_call('pkill -0 radosgw')['exit_code'] == 0
|
|
|
|
self.env.revert_snapshot("ready")
|
|
self.env.bootstrap_nodes(
|
|
self.env.d_env.nodes().slaves[:6])
|
|
|
|
cluster_id = self.fuel_web.create_cluster(
|
|
name=self.__class__.__name__,
|
|
mode=settings.DEPLOYMENT_MODE,
|
|
settings={
|
|
'volumes_lvm': False,
|
|
'volumes_ceph': True,
|
|
'images_ceph': True,
|
|
'objects_ceph': True,
|
|
'tenant': 'rados',
|
|
'user': 'rados',
|
|
'password': 'rados'
|
|
}
|
|
)
|
|
self.fuel_web.update_nodes(
|
|
cluster_id,
|
|
{
|
|
'slave-01': ['controller'],
|
|
'slave-02': ['controller'],
|
|
'slave-03': ['controller'],
|
|
'slave-04': ['compute', 'ceph-osd'],
|
|
'slave-05': ['compute', 'ceph-osd'],
|
|
'slave-06': ['compute', 'ceph-osd']
|
|
}
|
|
)
|
|
self.fuel_web.verify_network(cluster_id)
|
|
# Deploy cluster
|
|
self.fuel_web.deploy_cluster_wait(cluster_id)
|
|
|
|
# Network verification
|
|
self.fuel_web.verify_network(cluster_id)
|
|
|
|
# HAProxy backend checking
|
|
controller_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
|
|
cluster_id, ['controller'])
|
|
|
|
for node in controller_nodes:
|
|
logger.info("Check all HAProxy backends on {}".format(
|
|
node['meta']['system']['fqdn']))
|
|
haproxy_status = checkers.check_haproxy_backend(node['ip'])
|
|
assert_equal(haproxy_status['exit_code'], 1,
|
|
"HAProxy backends are DOWN. {0}".format(
|
|
haproxy_status))
|
|
|
|
self.fuel_web.check_ceph_status(cluster_id)
|
|
|
|
# Run ostf
|
|
self.fuel_web.run_ostf(cluster_id=cluster_id,
|
|
test_sets=['ha', 'smoke', 'sanity'])
|
|
|
|
# Check the radosgw daemon is started
|
|
with self.fuel_web.get_ssh_for_node('slave-01') as remote:
|
|
assert_true(radosgw_started(remote), 'radosgw daemon started')
|
|
|
|
self.env.make_snapshot("ceph_rados_gw")
|
|
|
|
|
|
@test(groups=["ceph_ha_one_controller", "ceph_migration"])
|
|
class VmBackedWithCephMigrationBasic(TestBasic):
|
|
"""VmBackedWithCephMigrationBasic.""" # TODO documentation
|
|
|
|
@test(depends_on=[SetupEnvironment.prepare_slaves_3],
|
|
groups=["ceph_migration"])
|
|
@log_snapshot_after_test
|
|
def migrate_vm_backed_with_ceph(self):
|
|
"""Check VM backed with ceph migration in ha mode with 1 controller
|
|
|
|
Scenario:
|
|
1. Create cluster
|
|
2. Add 1 node with controller and ceph OSD roles
|
|
3. Add 2 nodes with compute and ceph OSD roles
|
|
4. Deploy the cluster
|
|
5. Check ceph status
|
|
6. Run OSTF
|
|
7. Create a new VM, assign floating ip
|
|
8. Migrate VM
|
|
9. Check cluster and server state after migration
|
|
10. Terminate VM
|
|
11. Check that DHCP lease is not offered for MAC of deleted VM
|
|
12. Create a new VM for migration, assign floating ip
|
|
13. Create a volume and attach it to the VM
|
|
14. Create filesystem on the new volume and mount it to the VM
|
|
15. Migrate VM
|
|
16. Mount the volume after migration
|
|
17. Check cluster and server state after migration
|
|
18. Terminate VM
|
|
|
|
Duration 35m
|
|
Snapshot vm_backed_with_ceph_live_migration
|
|
"""
|
|
self.env.revert_snapshot("ready_with_3_slaves")
|
|
|
|
self.show_step(1, initialize=True)
|
|
|
|
cluster_id = self.fuel_web.create_cluster(
|
|
name=self.__class__.__name__,
|
|
mode=settings.DEPLOYMENT_MODE,
|
|
settings={
|
|
'volumes_ceph': True,
|
|
'images_ceph': True,
|
|
'ephemeral_ceph': True,
|
|
'volumes_lvm': False,
|
|
}
|
|
)
|
|
|
|
self.show_step(2)
|
|
self.show_step(3)
|
|
|
|
self.fuel_web.update_nodes(
|
|
cluster_id,
|
|
{
|
|
'slave-01': ['controller', 'ceph-osd'],
|
|
'slave-02': ['compute', 'ceph-osd'],
|
|
'slave-03': ['compute', 'ceph-osd']
|
|
}
|
|
)
|
|
creds = SSHAuth(username="cirros", password="test")
|
|
|
|
self.show_step(4)
|
|
|
|
# Cluster deploy
|
|
self.fuel_web.deploy_cluster_wait(cluster_id)
|
|
|
|
def _check():
|
|
# Run volume test several times with hope that it pass
|
|
test_path = ostf_test_mapping.OSTF_TEST_MAPPING.get(
|
|
'Create volume and attach it to instance')
|
|
logger.debug('Start to run test {0}'.format(test_path))
|
|
self.fuel_web.run_single_ostf_test(
|
|
cluster_id, test_sets=['smoke'],
|
|
test_name=test_path)
|
|
|
|
self.show_step(5)
|
|
try:
|
|
_check()
|
|
except AssertionError:
|
|
logger.debug(AssertionError)
|
|
logger.debug("Test failed from first probe,"
|
|
" we sleep 60 second try one more time "
|
|
"and if it fails again - test will fails ")
|
|
time.sleep(60)
|
|
_check()
|
|
|
|
self.show_step(6)
|
|
|
|
# Run ostf
|
|
self.fuel_web.run_ostf(cluster_id)
|
|
|
|
self.show_step(7)
|
|
|
|
# Create new server
|
|
os = os_actions.OpenStackActions(
|
|
self.fuel_web.get_public_vip(cluster_id))
|
|
net_name = self.fuel_web.get_cluster_predefined_networks_name(
|
|
cluster_id)['private_net']
|
|
|
|
logger.info("Create new server")
|
|
srv = os.create_server_for_migration(
|
|
neutron=True,
|
|
scenario='./fuelweb_test/helpers/instance_initial_scenario',
|
|
label=net_name)
|
|
logger.info("Srv is currently in status: {:s}".format(srv.status))
|
|
|
|
# Prepare to DHCP leases checks
|
|
net_name = self.fuel_web.get_cluster_predefined_networks_name(
|
|
cluster_id)['private_net']
|
|
srv_instance_ip = os.get_nova_instance_ip(srv, net_name=net_name)
|
|
srv_host_name = self.fuel_web.find_devops_node_by_nailgun_fqdn(
|
|
os.get_srv_hypervisor_name(srv),
|
|
self.env.d_env.nodes().slaves[:3]).name
|
|
net_id = os.get_network(net_name)['id']
|
|
ports = os.get_neutron_dhcp_ports(net_id)
|
|
dhcp_server_ip = ports[0]['fixed_ips'][0]['ip_address']
|
|
with self.fuel_web.get_ssh_for_node(srv_host_name) as srv_remote_node:
|
|
srv_instance_mac = os.get_instance_mac(srv_remote_node, srv)
|
|
|
|
logger.info("Assigning floating ip to server")
|
|
floating_ip = os.assign_floating_ip(srv)
|
|
srv_host = os.get_srv_host_name(srv)
|
|
logger.info("Server is on host {:s}".format(srv_host))
|
|
|
|
wait(lambda: tcp_ping(floating_ip.ip, 22), timeout=120,
|
|
timeout_msg='new VM ssh port ping timeout')
|
|
|
|
def ssh_ready(remote, ip, creds):
|
|
"""SSH Ready status
|
|
|
|
:type ip: str
|
|
:type creds: SSHAuth
|
|
"""
|
|
try:
|
|
remote.execute_through_host(ip, '/bin/true', creds)
|
|
return True
|
|
except paramiko.AuthenticationException:
|
|
logger.info("Authentication failed. Trying again in a minute.")
|
|
time.sleep(60)
|
|
return False
|
|
|
|
with self.fuel_web.get_ssh_for_node("slave-01") as remote:
|
|
wait(lambda: ssh_ready(remote, floating_ip.ip, creds), timeout=300)
|
|
md5before = remote.execute_through_host(
|
|
floating_ip.ip,
|
|
"md5sum {:s}".format("/home/test_file"),
|
|
auth=creds).stdout_str
|
|
|
|
self.show_step(8)
|
|
|
|
logger.info("Get available computes")
|
|
avail_hosts = os.get_hosts_for_migr(srv_host)
|
|
|
|
logger.info("Migrating server")
|
|
new_srv = os.migrate_server(srv, avail_hosts[0], timeout=200)
|
|
logger.info("Check cluster and server state after migration")
|
|
|
|
wait(lambda: tcp_ping(floating_ip.ip, 22), timeout=120,
|
|
timeout_msg='VM ssh port ping timeout after migration')
|
|
|
|
with self.fuel_web.get_ssh_for_node("slave-01") as remote:
|
|
md5after = remote.execute_through_host(
|
|
floating_ip.ip,
|
|
"md5sum {:s}".format("/home/test_file"),
|
|
auth=creds).stdout_str
|
|
|
|
checkers.diff_md5(md5before, md5after)
|
|
|
|
self.show_step(9)
|
|
|
|
with self.fuel_web.get_ssh_for_node("slave-01") as remote:
|
|
res = remote.execute_through_host(
|
|
floating_ip.ip,
|
|
"ping -q -c3 -w10 {0} | grep 'received' |"
|
|
" grep -v '0 packets received'"
|
|
.format(settings.PUBLIC_TEST_IP),
|
|
auth=creds)
|
|
logger.info("Ping {0} result on vm is: {1}"
|
|
.format(settings.PUBLIC_TEST_IP, res['stdout']))
|
|
|
|
logger.info("Check Ceph health is ok after migration")
|
|
self.fuel_web.check_ceph_status(cluster_id)
|
|
|
|
logger.info(
|
|
"Server is now on host {:s}".format(os.get_srv_host_name(new_srv)))
|
|
|
|
self.show_step(10)
|
|
|
|
logger.info("Terminate migrated server")
|
|
os.delete_instance(new_srv)
|
|
os.verify_srv_deleted(new_srv)
|
|
|
|
self.show_step(11)
|
|
# Check if the dhcp lease for instance still remains
|
|
# on the previous compute node. Related Bug: #1391010
|
|
_ip = self.fuel_web.get_nailgun_node_by_name('slave-01')['ip']
|
|
with self.fuel_web.get_ssh_for_node('slave-01') as remote:
|
|
dhcp_port_tag = ovs_get_tag_by_port(remote, ports[0]['id'])
|
|
assert_false(checkers.check_neutron_dhcp_lease(_ip,
|
|
srv_instance_ip,
|
|
srv_instance_mac,
|
|
dhcp_server_ip,
|
|
dhcp_port_tag),
|
|
"Instance has been deleted, but it's DHCP lease "
|
|
"for IP:{0} with MAC:{1} still offers by Neutron DHCP"
|
|
" agent.".format(srv_instance_ip,
|
|
srv_instance_mac))
|
|
self.show_step(12)
|
|
# Create a new server
|
|
logger.info("Create a new server for migration with volume")
|
|
srv = os.create_server_for_migration(
|
|
neutron=True,
|
|
scenario='./fuelweb_test/helpers/instance_initial_scenario',
|
|
label=net_name)
|
|
logger.info("Srv is currently in status: {:s}".format(srv.status))
|
|
|
|
logger.info("Assigning floating ip to server")
|
|
floating_ip = os.assign_floating_ip(srv)
|
|
srv_host = os.get_srv_host_name(srv)
|
|
logger.info("Server is on host {:s}".format(srv_host))
|
|
|
|
self.show_step(13)
|
|
logger.info("Create volume")
|
|
vol = os.create_volume()
|
|
logger.info("Attach volume to server")
|
|
os.attach_volume(vol, srv)
|
|
|
|
self.show_step(14)
|
|
wait(lambda: tcp_ping(floating_ip.ip, 22), timeout=120,
|
|
timeout_msg='new VM ssh port ping timeout')
|
|
logger.info("Create filesystem and mount volume")
|
|
|
|
with self.fuel_web.get_ssh_for_node("slave-01") as remote:
|
|
wait(lambda: ssh_ready(remote, floating_ip.ip, creds), timeout=300)
|
|
|
|
remote.execute_through_host(
|
|
floating_ip.ip,
|
|
'sudo sh /home/mount_volume.sh',
|
|
auth=creds)
|
|
|
|
remote.execute_through_host(
|
|
floating_ip.ip,
|
|
'sudo touch /mnt/file-on-volume',
|
|
auth=creds)
|
|
|
|
self.show_step(15)
|
|
logger.info("Get available computes")
|
|
avail_hosts = os.get_hosts_for_migr(srv_host)
|
|
|
|
logger.info("Migrating server")
|
|
new_srv = os.migrate_server(srv, avail_hosts[0], timeout=120)
|
|
|
|
logger.info("Check cluster and server state after migration")
|
|
wait(lambda: tcp_ping(floating_ip.ip, 22), timeout=120,
|
|
timeout_msg='VM ssh port ping timeout after migration')
|
|
|
|
self.show_step(16)
|
|
logger.info("Mount volume after migration")
|
|
with self.fuel_web.get_ssh_for_node("slave-01") as remote:
|
|
out = remote.execute_through_host(
|
|
floating_ip.ip,
|
|
'sudo mount /dev/vdb /mnt',
|
|
auth=creds)
|
|
|
|
logger.info("out of mounting volume is: {:s}".format(out['stdout']))
|
|
|
|
with self.fuel_web.get_ssh_for_node("slave-01") as remote:
|
|
out = remote.execute_through_host(
|
|
floating_ip.ip,
|
|
"sudo ls /mnt",
|
|
auth=creds)
|
|
assert_true("file-on-volume" in out['stdout'],
|
|
"File is absent in /mnt")
|
|
|
|
self.show_step(17)
|
|
logger.info("Check Ceph health is ok after migration")
|
|
self.fuel_web.check_ceph_status(cluster_id)
|
|
|
|
logger.info(
|
|
"Server is now on host {:s}".format(os.get_srv_host_name(new_srv)))
|
|
|
|
self.show_step(18)
|
|
logger.info("Terminate migrated server")
|
|
os.delete_instance(new_srv)
|
|
os.verify_srv_deleted(new_srv)
|
|
|
|
self.env.make_snapshot(
|
|
"vm_backed_with_ceph_live_migration")
|
|
|
|
|
|
@test(groups=["ceph_ha_one_controller", "ceph_partitions"])
|
|
class CheckCephPartitionsAfterReboot(TestBasic):
|
|
"""CheckCephPartitionsAfterReboot.""" # TODO documentation
|
|
|
|
@test(depends_on=[SetupEnvironment.prepare_slaves_3],
|
|
groups=["ceph_partitions"])
|
|
@log_snapshot_after_test
|
|
def check_ceph_partitions_after_reboot(self):
|
|
"""Check that Ceph OSD partitions are remounted after reboot
|
|
|
|
Scenario:
|
|
1. Create cluster in Ha mode with 1 controller
|
|
2. Add 1 node with controller role
|
|
3. Add 1 node with compute and Ceph OSD roles
|
|
4. Add 1 node with Ceph OSD role
|
|
5. Deploy the cluster
|
|
6. Check Ceph status
|
|
7. Read current partitions
|
|
8. Warm-reboot Ceph nodes
|
|
9. Read partitions again
|
|
10. Check Ceph health
|
|
11. Cold-reboot Ceph nodes
|
|
12. Read partitions again
|
|
13. Check Ceph health
|
|
|
|
Duration 40m
|
|
Snapshot check_ceph_partitions_after_reboot
|
|
|
|
"""
|
|
self.env.revert_snapshot("ready_with_3_slaves")
|
|
|
|
self.show_step(1, initialize=True)
|
|
|
|
cluster_id = self.fuel_web.create_cluster(
|
|
name=self.__class__.__name__,
|
|
mode=settings.DEPLOYMENT_MODE,
|
|
settings={
|
|
'volumes_ceph': True,
|
|
'images_ceph': True,
|
|
'osd_pool_size': '2',
|
|
'ephemeral_ceph': True,
|
|
'volumes_lvm': False,
|
|
}
|
|
)
|
|
|
|
self.show_step(2)
|
|
self.show_step(3)
|
|
self.show_step(4)
|
|
self.fuel_web.update_nodes(
|
|
cluster_id,
|
|
{
|
|
'slave-01': ['controller'],
|
|
'slave-02': ['compute', 'ceph-osd'],
|
|
'slave-03': ['ceph-osd']
|
|
}
|
|
)
|
|
|
|
self.show_step(5)
|
|
# Deploy cluster
|
|
self.fuel_web.deploy_cluster_wait(cluster_id)
|
|
|
|
self.show_step(6)
|
|
for node in ["slave-02", "slave-03"]:
|
|
|
|
self.show_step(7, node)
|
|
logger.info("Get partitions for {node}".format(node=node))
|
|
_ip = self.fuel_web.get_nailgun_node_by_name(node)['ip']
|
|
before_reboot_partitions = [utils.get_ceph_partitions(
|
|
_ip,
|
|
"/dev/vd{p}".format(p=part)) for part in ["b", "c"]]
|
|
|
|
self.show_step(8, node)
|
|
logger.info("Warm-restart nodes")
|
|
self.fuel_web.warm_restart_nodes(
|
|
[self.fuel_web.environment.d_env.get_node(name=node)])
|
|
|
|
self.show_step(9, node)
|
|
logger.info("Get partitions for {node} once again".format(
|
|
node=node
|
|
))
|
|
_ip = self.fuel_web.get_nailgun_node_by_name(node)['ip']
|
|
after_reboot_partitions = [utils.get_ceph_partitions(
|
|
_ip,
|
|
"/dev/vd{p}".format(p=part)) for part in ["b", "c"]]
|
|
|
|
if before_reboot_partitions != after_reboot_partitions:
|
|
logger.info("Partitions don`t match")
|
|
logger.info("Before reboot: "
|
|
"{:s}".format(before_reboot_partitions))
|
|
logger.info("After reboot: "
|
|
"{:s}".format(after_reboot_partitions))
|
|
raise Exception()
|
|
|
|
self.show_step(10, node)
|
|
logger.info("Check Ceph health is ok after reboot")
|
|
self.fuel_web.check_ceph_status(cluster_id)
|
|
|
|
self.show_step(11, node)
|
|
logger.info("Cold-restart nodes")
|
|
self.fuel_web.cold_restart_nodes(
|
|
[self.fuel_web.environment.d_env.get_node(name=node)])
|
|
|
|
self.show_step(12, node)
|
|
_ip = self.fuel_web.get_nailgun_node_by_name(node)['ip']
|
|
after_reboot_partitions = [utils.get_ceph_partitions(
|
|
_ip,
|
|
"/dev/vd{p}".format(p=part)) for part in ["b", "c"]]
|
|
|
|
if before_reboot_partitions != after_reboot_partitions:
|
|
logger.info("Partitions don`t match")
|
|
logger.info("Before reboot: "
|
|
"{:s}".format(before_reboot_partitions))
|
|
logger.info("After reboot: "
|
|
"{:s}".format(after_reboot_partitions))
|
|
raise Exception()
|
|
|
|
self.show_step(13, node)
|
|
logger.info("Check Ceph health is ok after reboot")
|
|
self.fuel_web.check_ceph_status(cluster_id)
|