fuel-qa/fuelweb_test/helpers/patching.py
Alexey Stepanov 79de91cdc0 Python3 code compatibility: Stage2.2 - API changes
six library usage is enabled for non transparent compatible code

blueprint fuel-qa-python3-compatibility

Change-Id: I2a636c72454809e59545ec20c3cb52124fd9753a
2016-03-28 11:32:52 +03:00

621 lines
26 KiB
Python

# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import re
import sys
import traceback
import zlib
from xml.dom.minidom import parseString
from proboscis import register
from proboscis import TestProgram
from proboscis.asserts import assert_equal
from proboscis.asserts import assert_is_not_none
from proboscis.asserts import assert_not_equal
from proboscis.asserts import assert_true
# pylint: disable=import-error,wrong-import-order
from six.moves.urllib.request import urlopen
from six.moves.urllib.parse import urlparse
# pylint: enable=import-error,wrong-import-order
import yaml
from fuelweb_test import logger
from fuelweb_test import settings
from fuelweb_test.helpers.ssh_manager import SSHManager
patching_validation_schema = {
'type': {
'required': True,
'values': ['service_stop', 'service_start', 'service_restart',
'server_down', 'server_up', 'server_reboot',
'run_command', 'upload_script', 'run_tasks'],
'data_type': str
},
'target': {
'required': True,
'values': {'master', 'slaves', 'controller_role', 'compute_role',
'cinder_role', 'ceph-osd_role', 'mongo_role',
'zabbix-server_role', 'base-os_role'},
'data_type': list
},
'service': {
'required': False,
'data_type': str
},
'command': {
'required': False,
'data_type': str
},
'script': {
'required': False,
'data_type': str
},
'upload_path': {
'required': False,
'data_type': str
},
'id': {
'required': True,
'data_type': int
},
'tasks': {
'required': False,
'data_type': list
},
'tasks_timeout': {
'required': False,
'data_type': int
},
}
def map_test(target):
assert_is_not_none(settings.PATCHING_BUG_ID,
"Bug ID wasn't specified, can't start patching tests!")
errata = get_errata(path=settings.PATCHING_APPLY_TESTS,
bug_id=settings.PATCHING_BUG_ID)
verify_errata(errata)
if not any(target == e_target['type'] for e_target in errata['targets']):
skip_patching_test(target, errata['target'])
env_distro = settings.OPENSTACK_RELEASE
master_distro = settings.OPENSTACK_RELEASE_CENTOS
if 'affected-pkgs' in errata.keys():
if target == 'master':
settings.PATCHING_PKGS = set(
[re.split('=|<|>', package)[0] for package
in errata['affected-pkgs'][master_distro.lower()]])
else:
settings.PATCHING_PKGS = set(
[re.split('=|<|>', package)[0] for package
in errata['affected-pkgs'][env_distro.lower()]])
available_env_packages = set()
available_master_packages = set()
for repo in settings.PATCHING_MIRRORS:
logger.debug(
'Checking packages from "{0}" repository'.format(repo))
available_env_packages.update(get_repository_packages(repo,
env_distro))
for repo in settings.PATCHING_MASTER_MIRRORS:
logger.debug(
'Checking packages from "{0}" repository'.format(repo))
available_master_packages.update(get_repository_packages(
repo, master_distro))
available_packages = available_env_packages | available_master_packages
if not settings.PATCHING_PKGS:
if target == 'master':
settings.PATCHING_PKGS = available_master_packages
else:
settings.PATCHING_PKGS = available_env_packages
else:
assert_true(settings.PATCHING_PKGS <= available_packages,
"Patching repositories don't contain all packages need"
"ed for tests. Need: {0}, available: {1}, missed: {2}."
"".format(settings.PATCHING_PKGS,
available_packages,
settings.PATCHING_PKGS - available_packages))
assert_not_equal(len(settings.PATCHING_PKGS), 0,
"No packages found in repository(s) for patching:"
" '{0} {1}'".format(settings.PATCHING_MIRRORS,
settings.PATCHING_MASTER_MIRRORS))
if target == 'master':
tests_groups = get_packages_tests(settings.PATCHING_PKGS,
master_distro,
target)
else:
tests_groups = get_packages_tests(settings.PATCHING_PKGS,
env_distro,
target)
if 'rally' in errata.keys():
if len(errata['rally']) > 0:
settings.PATCHING_RUN_RALLY = True
settings.RALLY_TAGS = errata['rally']
if settings.PATCHING_CUSTOM_TEST:
deployment_test = settings.PATCHING_CUSTOM_TEST
settings.PATCHING_SNAPSHOT = \
'patching_after_{0}'.format(deployment_test)
register(groups=['prepare_patching_environment'],
depends_on_groups=[deployment_test])
register(groups=['prepare_patching_master_environment'],
depends_on_groups=[deployment_test])
else:
program = TestProgram(argv=['none'])
deployment_test = None
for my_test in program.plan.tests:
if all(patching_group in my_test.entry.info.groups for
patching_group in tests_groups):
deployment_test = my_test
break
if deployment_test:
settings.PATCHING_SNAPSHOT = 'patching_after_{0}'.format(
deployment_test.entry.method.im_func.func_name)
if target == 'master':
register(groups=['prepare_patching_master_environment'],
depends_on=[deployment_test.entry.home])
else:
register(groups=['prepare_patching_environment'],
depends_on=[deployment_test.entry.home])
else:
raise Exception(
"Test with groups {0} not found.".format(tests_groups))
def get_repository_packages(remote_repo_url, repo_type):
repo_url = urlparse(remote_repo_url)
packages = []
if repo_type == settings.OPENSTACK_RELEASE_UBUNTU:
packages_url = '{0}/Packages'.format(repo_url.geturl())
pkgs_raw = urlopen(packages_url).read()
for pkg in pkgs_raw.split('\n'):
match = re.search(r'^Package: (\S+)\s*$', pkg)
if match:
packages.append(match.group(1))
else:
packages_url = '{0}/repodata/primary.xml.gz'.format(repo_url.geturl())
pkgs_xml = parseString(zlib.decompressobj(zlib.MAX_WBITS | 32).
decompress(urlopen(packages_url).read()))
for pkg in pkgs_xml.getElementsByTagName('package'):
packages.append(
pkg.getElementsByTagName('name')[0].firstChild.nodeValue)
return packages
def _get_target_and_project(_pkg, _all_pkgs):
for _installation_target in _all_pkgs.keys():
for _project in _all_pkgs[_installation_target]['projects']:
if _pkg in _project['packages']:
return _installation_target, _project['name']
def get_package_test_info(package, pkg_type, tests_path, patch_target):
packages_path = "{0}/{1}/packages.yaml".format(tests_path, pkg_type)
tests = set()
tests_file = 'test.yaml'
all_packages = yaml.load(open(packages_path).read())
assert_is_not_none(_get_target_and_project(package, all_packages),
"Package '{0}' doesn't belong to any installation "
"target / project".format(package))
target, project = _get_target_and_project(package, all_packages)
if patch_target == 'master':
if target not in ['master', 'bootstrap']:
return {None}
if patch_target == 'environment':
if target not in ['deployment', 'provisioning']:
return {None}
target_tests_path = "/".join((tests_path, pkg_type, target, tests_file))
project_tests_path = "/".join((tests_path, pkg_type, target, project,
tests_file))
package_tests_path = "/".join((tests_path, pkg_type, target, project,
package, tests_file))
for path in (target_tests_path, project_tests_path, package_tests_path):
try:
test = yaml.load(open(path).read())
if 'system_tests' in test.keys():
tests.update(test['system_tests']['tags'])
except IOError as e:
logger.warning('Ignoring exception: {!r}'.format(e))
logger.debug(traceback.format_exc())
return tests
def get_packages_tests(packages, distro, target):
assert_true(os.path.isdir(settings.PATCHING_PKGS_TESTS),
"Path for packages tests doesn't exist: '{0}'".format(
settings.PATCHING_PKGS_TESTS))
if distro == settings.OPENSTACK_RELEASE_UBUNTU:
pkg_type = 'deb'
else:
pkg_type = 'rpm'
packages_tests = set()
for package in packages:
tests = get_package_test_info(package,
pkg_type,
settings.PATCHING_PKGS_TESTS,
target)
assert_true(len(tests) > 0,
"Tests for package {0} not found".format(package))
if None in tests:
continue
packages_tests.update(tests)
return packages_tests
def mirror_remote_repository(admin_remote, remote_repo_url, local_repo_path):
repo_url = urlparse(remote_repo_url)
cut_dirs = len(repo_url.path.strip('/').split('/'))
download_cmd = ('wget --recursive --no-parent --no-verbose --reject "index'
'.html*,*.gif" --exclude-directories "{pwd}/repocache" '
'--directory-prefix {path} -nH --cut-dirs={cutd} {url}').\
format(pwd=repo_url.path.rstrip('/'), path=local_repo_path,
cutd=cut_dirs, url=repo_url.geturl())
result = admin_remote.execute(download_cmd)
assert_equal(result['exit_code'], 0, 'Mirroring of remote packages '
'repository failed: {0}'.format(
result))
def add_remote_repositories(environment, mirrors, prefix_name='custom_repo'):
repositories = set()
for mir in mirrors:
name = '{0}_{1}'.format(prefix_name, mirrors.index(mir))
local_repo_path = '/'.join([settings.PATCHING_WEB_DIR, name])
remote_repo_url = mir
with environment.d_env.get_admin_remote() as remote:
mirror_remote_repository(
admin_remote=remote,
remote_repo_url=remote_repo_url,
local_repo_path=local_repo_path)
repositories.add(name)
return repositories
def connect_slaves_to_repo(environment, nodes, repo_name):
repo_ip = environment.get_admin_node_ip()
repo_port = '8080'
repourl = 'http://{master_ip}:{repo_port}/{repo_name}/'.format(
master_ip=repo_ip, repo_name=repo_name, repo_port=repo_port)
if settings.OPENSTACK_RELEASE == settings.OPENSTACK_RELEASE_UBUNTU:
cmds = [
"echo -e '\ndeb {repourl} /' > /etc/apt/sources.list.d/{repo_name}"
".list".format(repourl=repourl, repo_name=repo_name),
"apt-key add <(curl -s '{repourl}/Release.key') || :".format(
repourl=repourl),
# Set highest priority to all repositories located on master node
"echo -e 'Package: *\nPin: origin {0}\nPin-Priority: 1060' > "
"/etc/apt/preferences.d/custom_repo".format(
environment.get_admin_node_ip()),
"apt-get update"
]
else:
cmds = [
"yum-config-manager --add-repo {url}".format(url=repourl),
"echo -e 'gpgcheck=0\npriority=20' >>/etc/yum.repos.d/{ip}_{port}_"
"{repo}_.repo".format(ip=repo_ip, repo=repo_name, port=repo_port),
"yum -y clean all",
]
for slave in nodes:
for cmd in cmds:
SSHManager().execute_on_remote(
ip=slave['ip'],
cmd=cmd
)
def connect_admin_to_repo(environment, repo_name):
repo_ip = environment.get_admin_node_ip()
repo_port = '8080'
repourl = 'http://{master_ip}:{repo_port}/{repo_name}/'.format(
master_ip=repo_ip, repo_name=repo_name, repo_port=repo_port)
cmds = [
"yum-config-manager --add-repo {url}".format(url=repourl),
"echo -e 'gpgcheck=0\npriority=20' >>/etc/yum.repos.d/{ip}_{port}_"
"{repo}_.repo".format(ip=repo_ip, repo=repo_name, port=repo_port),
"yum -y clean all",
# FIXME(apanchenko):
# Temporary disable this check in order to test packages update
# inside Docker containers. When building of new images for containers
# is implemented, we should check here that `yum check-update` returns
# ONLY `100` exit code (updates are available for master node).
"yum check-update; [[ $? -eq 100 || $? -eq 0 ]]"
]
for cmd in cmds:
SSHManager().execute_on_remote(
ip=SSHManager().admin_ip,
cmd=cmd
)
def update_packages(environment, remote, packages, exclude_packages=None):
if settings.OPENSTACK_RELEASE == settings.OPENSTACK_RELEASE_UBUNTU:
cmds = [
'apt-get -o Dpkg::Options::="--force-confdef" '
'-o Dpkg::Options::="--force-confold" -y install '
'--only-upgrade {0}'.format(' '.join(packages))
]
if exclude_packages:
exclude_commands = ["apt-mark hold {0}".format(pkg)
for pkg in exclude_packages]
cmds = exclude_commands + cmds
else:
cmds = [
"yum -y update --nogpgcheck {0} -x '{1}'".format(
' '.join(packages), ','.join(exclude_packages or []))
]
for cmd in cmds:
environment.execute_remote_cmd(remote, cmd, exit_code=0)
def update_packages_on_slaves(environment, slaves, packages=None,
exclude_packages=None):
if not packages:
# Install all updates
packages = ' '
for slave in slaves:
with environment.d_env.get_ssh_to_remote(slave['ip']) as remote:
update_packages(environment, remote, packages, exclude_packages)
def get_slaves_ips_by_role(slaves, role=None):
if role:
return [slave['ip'] for slave in slaves if role in slave['roles']]
return [slave['ip'] for slave in slaves]
def get_devops_slaves_by_role(env, slaves, role=None):
if role:
return [env.fuel_web.find_devops_node_by_nailgun_fqdn(slave['fqdn'],
env.d_env.nodes().slaves)
for slave in slaves if role in slave['roles']]
return [env.fuel_web.find_devops_node_by_nailgun_fqdn(slave['fqdn'],
env.d_env.nodes().slaves) for slave in slaves]
def get_slaves_ids_by_role(slaves, role=None):
if role:
return [slave['id'] for slave in slaves if role in slave['roles']]
return [slave['id'] for slave in slaves]
def verify_fix_apply_step(apply_step):
validation_schema = patching_validation_schema
for key in validation_schema.keys():
if key in apply_step.keys():
is_exists = apply_step[key] is not None
else:
is_exists = False
if validation_schema[key]['required']:
assert_true(is_exists, "Required field '{0}' not found in patch "
"apply scenario step".format(key))
if not is_exists:
continue
is_valid = True
if 'values' in validation_schema[key].keys():
if validation_schema[key]['data_type'] == str:
is_valid = apply_step[key] in validation_schema[key]['values']
elif validation_schema[key]['data_type'] in (list, set):
is_valid = set(apply_step[key]) <= \
validation_schema[key]['values']
assert_true(is_valid, 'Step in patch apply actions scenario '
'contains incorrect data: "{key}": "{value}"'
'. Supported values for "{key}" are '
'"{valid}"'.format(
key=key,
value=apply_step[key],
valid=validation_schema[key]['values']))
if 'data_type' in validation_schema[key].keys():
assert_true(
isinstance(
apply_step[key], validation_schema[key]['data_type']),
"Unexpected data type in patch apply scenario step: '"
"{key}' is '{type}', but expecting '{expect}'.".format(
key=key,
type=type(apply_step[key]),
expect=validation_schema[key]['data_type']))
def validate_fix_apply_step(apply_step, environment, slaves):
verify_fix_apply_step(apply_step)
slaves = [] if not slaves else slaves
command = '',
remotes_ips = set()
devops_action = ''
devops_nodes = set()
nodes_ids = set()
if apply_step['type'] == 'run_tasks':
remotes_ips.add(environment.get_admin_node_ip())
assert_true('master' not in apply_step['target'],
"Action type 'run_tasks' accepts only slaves (roles) "
"as target value, but 'master' is specified!")
for target in apply_step['target']:
if target == 'slaves':
nodes_ids.update(get_slaves_ids_by_role(slaves, role=None))
else:
role = target.split('_role')[0]
nodes_ids.update(get_slaves_ids_by_role(slaves, role=role))
else:
for target in apply_step['target']:
if target == 'master':
remotes_ips.add(environment.get_admin_node_ip())
devops_nodes.add(
environment.d_env.nodes().admin)
elif target == 'slaves':
remotes_ips.update(get_slaves_ips_by_role(slaves, role=None))
devops_nodes.update(get_devops_slaves_by_role(environment,
slaves))
else:
role = target.split('_role')[0]
remotes_ips.update(get_slaves_ips_by_role(slaves, role))
devops_nodes.update(get_devops_slaves_by_role(environment,
slaves,
role=role))
if apply_step['type'] in ('service_stop', 'service_start',
'service_restart'):
assert_true(len(apply_step['service'] or '') > 0,
"Step #{0} in apply patch scenario perform '{1}', but "
"service isn't specified".format(apply_step['id'],
apply_step['type']))
action = apply_step['type'].split('service_')[1]
command = (
"find /etc/init.d/ -regex '/etc/init.d/{service}' -printf "
"'%f\n' -quit | xargs -i service {{}} {action}".format(
service=apply_step['service'], action=action), )
elif apply_step['type'] in ('server_down', 'server_up', 'server_reboot'):
assert_true('master' not in apply_step['target'],
'Action type "{0}" doesn\'t accept "master" node as '
'target! Use action "run_command" instead.'.format(
apply_step['type']))
devops_action = apply_step['type'].split('server_')[1]
elif apply_step['type'] == 'upload_script':
assert_true(len(apply_step['script'] or '') > 0,
"Step #{0} in apply patch scenario perform '{1}', but "
"script isn't specified".format(apply_step['id'],
apply_step['type']))
assert_true(len(apply_step['upload_path'] or '') > 0,
"Step #{0} in apply patch scenario perform '{1}', but "
"upload path isn't specified".format(apply_step['id'],
apply_step['type']))
command = ('UPLOAD', apply_step['script'], apply_step['upload_path'])
elif apply_step['type'] == 'run_tasks':
assert_true(len(apply_step['tasks'] or '') > 0,
"Step #{0} in apply patch scenario perform '{1}', but "
"tasks aren't specified".format(apply_step['id'],
apply_step['type']))
tasks_timeout = apply_step['tasks_timeout'] if 'tasks_timeout' in \
apply_step.keys() else 60 * 30
command = (
'RUN_TASKS',
nodes_ids,
apply_step['tasks'],
tasks_timeout
)
else:
assert_true(len(apply_step['command'] or '') > 0,
"Step #{0} in apply patch scenario perform '{1}', but "
"command isn't specified".format(apply_step['id'],
apply_step['type']))
command = apply_step['command']
# remotes sessions .clear() placed in run_actions()
remotes = [environment.d_env.get_ssh_to_remote(ip) for ip in remotes_ips] \
if command else []
devops_nodes = devops_nodes if devops_action else []
return command, remotes, devops_action, devops_nodes
def get_errata(path, bug_id):
scenario_path = '{0}/bugs/{1}/erratum.yaml'.format(path, bug_id)
assert_true(os.path.exists(scenario_path),
"Erratum for bug #{0} is not found in '{1}' "
"directory".format(bug_id, settings.PATCHING_APPLY_TESTS))
with open(scenario_path) as f:
return yaml.load(f.read())
def verify_errata(errata):
actions_types = ('patch-scenario', 'verify-scenario')
distro = settings.OPENSTACK_RELEASE.lower()
for target in errata['targets']:
for action_type in actions_types:
assert_true(distro in target[action_type].keys(),
"Steps for '{0}' not found for '{1}' distro!".format(
action_type, distro))
scenario = sorted(target[action_type][distro],
key=lambda k: k['id'])
for step in scenario:
verify_fix_apply_step(step)
def run_actions(environment, target, slaves, action_type='patch-scenario'):
errata = get_errata(path=settings.PATCHING_APPLY_TESTS,
bug_id=settings.PATCHING_BUG_ID)
distro = settings.OPENSTACK_RELEASE.lower()
target_scenarios = [e_target for e_target in errata['targets']
if target == e_target['type']]
assert_true(len(target_scenarios) > 0,
"Can't found patch scenario for '{0}' target in erratum "
"for bug #{1}!".format(target, settings.PATCHING_BUG_ID))
scenario = sorted(target_scenarios[0][action_type][distro],
key=lambda k: k['id'])
for step in scenario:
command, remotes, devops_action, devops_nodes = \
validate_fix_apply_step(step, environment, slaves)
if 'UPLOAD' in command:
file_name = command[1]
upload_path = command[2]
source_path = '{0}/bugs/{1}/tests/{2}'.format(
settings.PATCHING_APPLY_TESTS,
settings.PATCHING_BUG_ID,
file_name)
assert_true(os.path.exists(source_path),
'File for uploading "{0}" doesn\'t exist!'.format(
source_path))
for remote in remotes:
remote.upload(source_path, upload_path)
continue
elif 'RUN_TASKS' in command:
nodes_ids = command[1]
tasks = command[2]
timeout = command[3]
nodes = [node for node in environment.fuel_web.client.list_nodes()
if node['id'] in nodes_ids]
assert_true(len(nodes_ids) == len(nodes),
'Get nodes with ids: {0} for deployment task, but '
'found {1}!'.format(nodes_ids,
[n['id'] for n in nodes]))
assert_true(len(set([node['cluster'] for node in nodes])) == 1,
'Slaves for patching actions belong to different '
'environments, can\'t run deployment tasks!')
cluster_id = nodes[0]['cluster']
environment.fuel_web.wait_deployment_tasks(cluster_id, nodes_ids,
tasks, timeout)
continue
for remote in remotes:
environment.execute_remote_cmd(remote, command)
if devops_action == 'down':
environment.fuel_web.warm_shutdown_nodes(devops_nodes)
elif devops_action == 'up':
environment.fuel_web.warm_start_nodes(devops_nodes)
elif devops_action == 'reboot':
environment.fuel_web.warm_restart_nodes(devops_nodes)
# clear connections
for remote in remotes:
remote.clear()
def apply_patches(environment, target, slaves=None):
run_actions(environment, target, slaves, action_type='patch-scenario')
def verify_fix(environment, target, slaves=None):
run_actions(environment, target, slaves, action_type='verify-scenario')
def skip_patching_test(target, errata_target):
# TODO(apanchenko):
# If 'target' from erratum doesn't match 'target' from tests we need to
# skip tests and return special exit code, so Jenkins is able to recognize
# test were skipped and it shouldn't vote to CRs (just leave comment)
logger.error('Tests for "{0}" were started, but patches are targeted to '
'"{1}" according to erratum.'.format(target, errata_target))
sys.exit(123)