openstack/fuel-qa
Code Issues Proposed changes
c85e75fdc4
fuel-qa/fuelweb_test/tests/tests_upgrade/upgrade_base.py
Alexey Stepanov 3be097da19 Call upgrade_release for cluster upgrade tests 
Call upgrade_release for cluster upgrade tests:
 fuel2 release clone $ORIG_ID $RELEASE_ID -f value -c id

Change-Id: I0a010bdce97a754982de4cd734a32fb74ab15067
(cherry picked from commit 7232656)
2016-08-31 11:26:58 +03:00

400 lines
16 KiB
Python
Raw Blame History

# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import unicode_literals
# pylint: disable=import-error
# pylint: disable=no-name-in-module
from distutils.version import LooseVersion
# pylint: enable=no-name-in-module
# pylint: enable=import-error
from proboscis.asserts import assert_equal
from proboscis.asserts import assert_not_equal
from proboscis.asserts import assert_true
from proboscis import SkipTest
import six
from fuelweb_test import logger
from fuelweb_test.settings import KEYSTONE_CREDS
from fuelweb_test.settings import OPENSTACK_RELEASE
from fuelweb_test.settings import OPENSTACK_RELEASE_UBUNTU
from fuelweb_test.settings import UPGRADE_FUEL_FROM
from fuelweb_test.settings import UPGRADE_FUEL_TO
from fuelweb_test.tests.tests_upgrade.test_data_driven_upgrade_base import \
DataDrivenUpgradeBase
class OSUpgradeBase(DataDrivenUpgradeBase):
def __init__(self):
self.__old_cluster_name = None
super(OSUpgradeBase, self).__init__()
@property
def old_cluster_name(self):
return self.__old_cluster_name
@old_cluster_name.setter
def old_cluster_name(self, new_name):
if not isinstance(new_name, (six.string_types, six.text_type)):
logger.error('old_cluster_name === {!r}'.format(new_name))
raise TypeError('{!r} is not string'.format(new_name))
self.__old_cluster_name = new_name
@staticmethod
def check_release_requirements():
if OPENSTACK_RELEASE_UBUNTU not in OPENSTACK_RELEASE:
raise SkipTest('{0} not in {1}'.format(
OPENSTACK_RELEASE_UBUNTU, OPENSTACK_RELEASE))
def minimal_check(self, seed_cluster_id, nwk_check=False):
if nwk_check:
self.show_step(self.next_step)
self.fuel_web.verify_network(seed_cluster_id)
self.show_step(self.next_step)
self.fuel_web.run_single_ostf_test(
cluster_id=seed_cluster_id, test_sets=['sanity'],
test_name=('fuel_health.tests.sanity.test_sanity_identity'
'.SanityIdentityTest.test_list_users'))
def check_ceph_health(self, ip):
ceph_health = self.ssh_manager.check_call(
ip=ip, command="ceph health").stdout_str
# There are an issue with PG calculation - LP#1464656
try:
assert_true("HEALTH_OK" in ceph_health,
"Ceph health is not ok! Inspect output below:\n"
"{!r}".format(ceph_health))
except AssertionError:
logger.warning("Ceph health is not ok! trying to check LP#1464656")
if "HEALTH_WARN" in ceph_health and "too many PGs per OSD" in \
ceph_health:
logger.info("Known issue in ceph - see LP#1464656 for details")
else:
raise
@property
def orig_cluster_id(self):
"""Get cluster id for old_cluster_name
:rtype: int
"""
if self.old_cluster_name is None:
raise RuntimeError('old_cluster_name is not set')
return self.fuel_web.client.get_cluster_id(self.old_cluster_name)
def upgrade_release(self, use_net_template=False):
self.show_step(self.next_step)
if not use_net_template:
return int(
self.ssh_manager.check_call(
ip=self.env.get_admin_node_ip(),
command='fuel2 release clone {0} {1} '
'-f value -c id'.format(
self.orig_cluster_id,
self.fuel_web.client.get_release_id()
),
error_info='RELEASE_ID clone failed'
).stdout_str
)
else:
raise NotImplementedError(
'Upgrade with network templates is not supported now')
def upgrade_env_code(self, release_id):
self.show_step(self.next_step)
seed_id = int(
self.ssh_manager.check_call(
ip=self.env.get_admin_node_ip(),
command="octane upgrade-env {0} {1}".format(
self.orig_cluster_id,
release_id
),
error_info="'upgrade-env' command failed, "
"inspect logs for details"
).stdout_str)
new_cluster_id = int(self.fuel_web.get_last_created_cluster())
assert_not_equal(
self.orig_cluster_id, seed_id,
"Cluster IDs are the same: old={} and new={}".format(
self.orig_cluster_id, seed_id))
assert_equal(
seed_id,
new_cluster_id,
"Cluster ID was changed, but it's not the last:"
" abnormal activity or configuration error presents!\n"
"\tSEED ID: {}\n"
"\tLAST ID: {}".format(seed_id, new_cluster_id)
)
cluster_release_id = int(
self.fuel_web.get_cluster_release_id(seed_id)
)
assert_equal(
cluster_release_id,
release_id,
"Release ID {} is not equals to expected {}".format(
cluster_release_id,
release_id
)
)
def upgrade_first_controller_code(self, seed_cluster_id):
self.show_step(self.next_step)
controller = self.fuel_web.get_devops_node_by_nailgun_node(
self.fuel_web.get_nailgun_cluster_nodes_by_roles(
self.orig_cluster_id, ["controller"])[0])
primary = self.fuel_web.get_nailgun_node_by_devops_node(
self.fuel_web.get_nailgun_primary_node(controller)
)
self.show_step(self.next_step)
self.ssh_manager.check_call(
ip=self.ssh_manager.admin_ip,
command="octane upgrade-node --isolated "
"{0} {1}".format(seed_cluster_id, primary["id"]),
error_info="octane upgrade-node failed")
self.show_step(self.next_step)
tasks_started_by_octane = [
task for task in self.fuel_web.client.get_tasks()
if task['cluster'] == seed_cluster_id]
for task in tasks_started_by_octane:
self.fuel_web.assert_task_success(task)
self.show_step(self.next_step)
self.minimal_check(seed_cluster_id=seed_cluster_id)
def upgrade_db_code(self, seed_cluster_id):
self.show_step(self.next_step)
orig_controller = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
self.orig_cluster_id, ["controller"])[0]
seed_controller = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
seed_cluster_id, ["controller"])[0]
mysql_req = (
'mysql cinder <<< "select id from volumes;"; '
'mysql glance <<< "select id from images"; '
'mysql neutron <<< "(select id from networks) '
'UNION (select id from routers) '
'UNION (select id from subnets)"; '
'mysql keystone <<< "(select id from project) '
'UNION (select id from user)"')
self.show_step(self.next_step)
target_ids = self.ssh_manager.check_call(
ip=orig_controller["ip"], command=mysql_req).stdout
self.show_step(self.next_step)
self.ssh_manager.check_call(
ip=self.ssh_manager.admin_ip,
command="octane upgrade-db {0} {1}".format(
self.orig_cluster_id, seed_cluster_id),
error_info="octane upgrade-db failed")
self.show_step(self.next_step)
crm_status = self.ssh_manager.check_call(
ip=seed_controller["ip"], command="crm resource status").stdout
while crm_status:
current = crm_status.pop(0)
if "vip" in current:
assert_true("Started" in current)
elif "master_p" in current:
next_element = crm_status.pop(0)
assert_true("Masters: [ node-" in next_element)
elif any(x in current for x in ["ntp", "mysql", "dns"]):
next_element = crm_status.pop(0)
assert_true("Started" in next_element)
elif any(x in current for x in ["nova", "cinder", "keystone",
"heat", "neutron", "glance"]):
next_element = crm_status.pop(0)
assert_true("Stopped" in next_element)
seed_ids = self.ssh_manager.check_call(
ip=seed_controller["ip"], command=mysql_req).stdout
assert_equal(sorted(target_ids), sorted(seed_ids),
"Objects in target and seed dbs are different")
def upgrade_ceph_code(self, seed_cluster_id):
seed_controller = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
seed_cluster_id, ["controller"])[0]
self.show_step(self.next_step)
self.ssh_manager.check_call(
ip=self.ssh_manager.admin_ip,
command="octane upgrade-ceph {0} {1}".format(
self.orig_cluster_id, seed_cluster_id),
error_info="octane upgrade-ceph failed")
self.show_step(self.next_step)
self.check_ceph_health(seed_controller['ip'])
def upgrade_control_plane_code(self, seed_cluster_id):
self.show_step(self.next_step)
self.ssh_manager.check_call(
ip=self.ssh_manager.admin_ip,
command="octane upgrade-control {0} {1}".format(
self.orig_cluster_id, seed_cluster_id),
error_info="octane upgrade-control failed")
self.show_step(self.next_step)
controllers = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
seed_cluster_id, ["controller"])
old_controllers = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
self.orig_cluster_id, ["controller"])
old_computes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
self.orig_cluster_id, ["compute"])
def collect_management_ips(node_list):
result = []
for item in node_list:
for data in item["network_data"]:
if data["name"] == "management":
result.append(data["ip"].split("/")[0])
return result
ping_ips = collect_management_ips(controllers + old_computes)
ping_ips.append(self.fuel_web.get_mgmt_vip(seed_cluster_id))
non_ping_ips = collect_management_ips(old_controllers)
ping_cmd = "ping -W 1 -i 1 -s 56 -c 1 -w 10 {host}"
for node in controllers + old_computes:
self.ssh_manager.check_call(
ip=node["ip"], command="ip -s -s neigh flush all")
for ip in ping_ips:
self.ssh_manager.check_call(
ip=node["ip"],
command=ping_cmd.format(host=ip),
error_info="Can not ping {0} from {1}"
"need to check network"
" connectivity".format(ip, node["ip"]))
for ip in non_ping_ips:
self.ssh_manager.check_call(
ip=node["ip"],
command=ping_cmd.format(host=ip),
error_info="Patch ports from old controllers wasn't "
"removed",
expected=[1, 2]) # No reply, Other errors
crm = self.ssh_manager.check_call(
ip=controllers[0]["ip"],
command="crm resource status").stdout
while crm:
current = crm.pop(0)
if "vip" in current:
assert_true("Started" in current)
elif "master_p" in current:
next_element = crm.pop(0)
assert_true("Masters: [ node-" in next_element)
elif any(x in current for x in ["ntp", "mysql", "dns",
"nova", "cinder", "keystone",
"heat", "neutron", "glance"]):
next_element = crm.pop(0)
assert_true("Started" in next_element)
def upgrade_controllers_code(self, seed_cluster_id):
self.show_step(self.next_step)
old_controllers = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
self.orig_cluster_id, ["controller"])
self.ssh_manager.check_call(
ip=self.ssh_manager.admin_ip,
command="octane upgrade-node {0} {1}".format(
seed_cluster_id,
" ".join([str(ctrl["id"]) for ctrl in old_controllers])),
error_info="octane upgrade-node failed")
self.show_step(self.next_step)
tasks_started_by_octane = [
task for task in self.fuel_web.client.get_tasks()
if task['cluster'] == seed_cluster_id]
for task in tasks_started_by_octane:
self.fuel_web.assert_task_success(task)
self.minimal_check(seed_cluster_id=seed_cluster_id, nwk_check=True)
def upgrade_ceph_osd_code(self, seed_cluster_id):
seed_controller = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
seed_cluster_id, ["controller"]
)[0]
self.show_step(self.next_step)
self.ssh_manager.check_call(
ip=self.ssh_manager.admin_ip,
command="octane upgrade-osd --admin-password {0} {1}".format(
KEYSTONE_CREDS['password'],
self.orig_cluster_id),
error_info="octane upgrade-osd failed"
)
self.show_step(self.next_step)
self.check_ceph_health(seed_controller['ip'])
self.minimal_check(seed_cluster_id=seed_cluster_id, nwk_check=True)
def pre_upgrade_computes(self, orig_cluster_id, seed_cluster_id):
self.show_step(self.next_step)
# Fuel-octane can run pre-upgrade only starting from version 9.0 and
# we are upgrading packages only if version difference is >1 step
if LooseVersion(UPGRADE_FUEL_TO) >= LooseVersion('9.0') and \
LooseVersion(UPGRADE_FUEL_FROM) < LooseVersion('8.0'):
computes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
orig_cluster_id, ["compute"]
)
self.ssh_manager.check_call(
ip=self.ssh_manager.admin_ip,
command="octane upgrade-compute {0} {1}".format(
seed_cluster_id,
" ".join([str(comp["id"]) for comp in computes])),
error_info="octane upgrade-node failed")
def upgrade_nodes(self, seed_cluster_id, nodes_str, live_migration=False):
self.ssh_manager.check_call(
ip=self.ssh_manager.admin_ip,
command=(
"octane upgrade-node {migration} {seed_cluster_id} "
"{nodes!s}".format(
migration='' if live_migration else '--no-live-migration',
seed_cluster_id=seed_cluster_id,
nodes=nodes_str)),
error_info="octane upgrade-node failed")
def clean_up(self, seed_cluster_id):
self.show_step(self.next_step)
self.ssh_manager.check_call(
ip=self.ssh_manager.admin_ip,
command="octane cleanup {0}".format(seed_cluster_id),
error_info="octane cleanup cmd failed")
View Git Blame Copy Permalink