fuel-qa/fuelweb_test/tests/test_cgroups.py
slava 8757f4b3bd Rerun set of tasks to update hiera info before start 'cgroups' task
Currently we are not collecting task's dependencies when
we run it separately. So, in this case only 'cgroups' task
will be triggered with outdated hiera data.

Change-Id: I671c011fa6e51acabaf64e9ba37a7e303b506543
Closes-Bug: #1573066
2016-05-12 22:42:17 +03:00

483 lines
17 KiB
Python

# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import division
import re
import json
from proboscis import test
from proboscis import asserts
from fuelweb_test.helpers.decorators import log_snapshot_after_test
from fuelweb_test import settings
from fuelweb_test import logger
from fuelweb_test.tests.base_test_case import SetupEnvironment
from fuelweb_test.tests.base_test_case import TestBasic
from fuelweb_test.helpers import utils
@test(groups=["cgroup_ha"])
class TestCgroupHa(TestBasic):
"""Tests for verification deployment with enabled cgroup."""
@staticmethod
def generate_attributes(cgroups):
"""Generate cluster attributes structure from cgroups dicts."""
attributes = {}
for cgroup in cgroups:
if "limit" not in cgroup:
limit = {}
else:
limit = {cgroup["limit"]: cgroup["value"]}
attributes = utils.dict_merge(attributes, {
cgroup["process"]: {
"label": cgroup["process"],
"type": "text",
"value": {
cgroup["controller"]: limit
}
}
})
for cgroup in attributes.values():
cgroup["value"] = json.dumps(cgroup["value"])
return {"editable": {"cgroups": attributes}}
@staticmethod
def check_cgconfig_setup(config, process, controller,
limit=None, value=None):
"""Check /etc/cgconfig.conf contains properly configured cgroup."""
actual_limit = config[process][controller]
if limit is None and value is None:
asserts.assert_equal(actual_limit, {},
"Actual limit is not empty: {}"
.format(actual_limit))
else:
asserts.assert_equal(actual_limit[limit], value,
"Actual value limit is not as expected for "
"process {}, controller {}, limit {}, "
"expected value = {}, actual == {}"
.format(process, controller, limit, value,
actual_limit[limit]))
@staticmethod
def generate_lscgroups(cgroups):
"""Generate a list of lscgroups entities from cgroups dicts."""
return ["{}:/{}".format(cgroup["controller"], cgroup["process"])
for cgroup in cgroups]
def apply_cgroups(self, cgroups, node_ids):
cluster_id = self.fuel_web.get_last_created_cluster()
self.fuel_web.client.update_cluster_attributes(
cluster_id, self.generate_attributes(cgroups))
task = self.fuel_web.client.put_deployment_tasks_for_cluster(
cluster_id=cluster_id,
data=["upload_configuration", "configuration_symlink",
"hiera", "cgroups"],
node_id=node_ids)
self.fuel_web.assert_task_success(task)
def get_cgroups_config(self, nailgun_node):
"""Get /etc/cgconfig.conf from node, transform it to json and loads
Before transformation:
group mysqld {
memory {
memory.swappiness = 0;
}
}
group keystone {
cpu {
cpu.shares = 70;
}
}
group rabbitmq {
blkio {
blkio.weight = 500;
}
memory {
memory.swappiness = 0;
}
}
After transformation:
{
"mysqld": {
"memory": {
"memory.swappiness": 0
}
},
"keystone": {
"cpu": {
"cpu.shares": 70
}
},
"rabbitmq": {
"blkio": {
"blkio.weight": 500
},
"memory": {
"memory.swappiness": 0
}
}
}
"""
cmd = "cat /etc/cgconfig.conf"
result = self.ssh_manager.execute(nailgun_node['ip'], cmd)["stdout"]
cgroups_config = "".join([line for line in result
if not line.startswith("#")])
cgroups_to_json = [
('group ', ''), # Remove group tag
(' {', ': {'), # Replace { -> : {
('}', '},'), # Replace } -> },
(';', ','), # Replace ; -> ,
(' = ', ': '), # Replace = -> :
('[a-z_]+\.{0,1}[a-z_]*', '"\g<0>"'), # Wrap all words with " "
# Words could contain period
('[\s\S]*', '{\g<0> }'), # Wrap whole string with {}
(',[ \t\r\n]+}', '}') # Clear trailing commas
]
for pattern, replace in cgroups_to_json:
cgroups_config = re.sub(pattern, replace, cgroups_config)
return json.loads(cgroups_config)
def check_cgroups_on_node(self, nailgun_node, cgroups):
"""Complex validation of cgroups on particular node."""
cgroups_config = self.get_cgroups_config(nailgun_node)
for cgroup in cgroups:
logger.info("Check cgroup config for {} {} on node {}"
.format(cgroup["process"], cgroup["controller"],
nailgun_node['fqdn']))
self.check_cgconfig_setup(config=cgroups_config, **cgroup)
for lscgroup in self.generate_lscgroups(cgroups):
check_group_cmd = 'sudo lscgroup | fgrep -q {}'
logger.info('Check {} group existence on controller node {}'
.format(lscgroup, nailgun_node['fqdn']))
self.ssh_manager.check_call(nailgun_node['ip'],
check_group_cmd.format(lscgroup))
for cgroup in cgroups:
check_rule_cmd = ("fgrep {} /etc/cgrules.conf | fgrep -q {}"
.format(cgroup["process"], cgroup["controller"]))
logger.info("Check cgrule {} {} on controller node {}"
.format(cgroup["process"], cgroup["controller"],
nailgun_node['fqdn']))
self.ssh_manager.check_call(nailgun_node['ip'], check_rule_cmd)
@test(depends_on=[SetupEnvironment.prepare_slaves_5],
groups=['deploy_ha_cgroup'])
@log_snapshot_after_test
def deploy_ha_cgroup(self):
"""Deploy cluster in HA mode with enabled cgroups
Scenario:
1. Create cluster
2. Add 3 nodes with controller role
3. Add 1 node with compute role
4. Add 1 node with cinder role
5. Deploy the cluster
6. Check ceph status
7. Run OSTF
Duration 90m
Snapshot deploy_ha_cgroup
"""
self.check_run("deploy_ha_cgroup")
self.env.revert_snapshot("ready_with_5_slaves")
data = {
'tenant': 'cgroup',
'user': 'cgroup',
'password': 'cgroup',
'net_provider': 'neutron',
'net_segment_type': settings.NEUTRON_SEGMENT['vlan']
}
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=settings.DEPLOYMENT_MODE,
settings=data)
self.fuel_web.update_nodes(
cluster_id,
{
'slave-01': ['controller'],
'slave-02': ['controller'],
'slave-03': ['controller'],
'slave-04': ['compute'],
'slave-05': ['cinder']
}
)
cgroup_data = [{
"process": "keystone",
"controller": "cpu",
"limit": "cpu.shares",
"value": 70,
}]
self.fuel_web.client.update_cluster_attributes(
cluster_id, self.generate_attributes(cgroup_data))
# Cluster deploy
self.fuel_web.deploy_cluster_wait(cluster_id)
# Run ostf
self.fuel_web.run_ostf(cluster_id=cluster_id)
# Check that task cgroup was executed
cmd = 'fgrep "MODULAR: cgroups/cgroups.pp" -q /var/log/puppet.log'
n_ctrls = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
cluster_id, ['controller'])
for nailgun_node in n_ctrls:
logger.info('Check cgroups task on controller node {0}'.format(
nailgun_node["fqdn"]))
self.ssh_manager.check_call(nailgun_node['ip'], cmd)
self.check_cgroups_on_node(nailgun_node, cgroup_data)
self.env.make_snapshot("deploy_ha_cgroup", is_make=True)
@test(depends_on=[deploy_ha_cgroup],
groups=['apply_cgroups_after_deploy'])
@log_snapshot_after_test
def apply_cgroups_after_deploy(self):
"""Apply, reconfigure and disable cgroups limits to services
Scenario:
1. Revert snapshot deploy_ha_cgroup
2. Configure and validate cgroups for mysqld, rabbitmq
and keystone
3. Reconfigure and validate cgroups for mysqld,
rabbitmq and keystone
4. Disable cgroups for mysqld, rabbitmq and keystone
Duration 15m
"""
self.show_step(1)
self.env.revert_snapshot("deploy_ha_cgroup")
cluster_id = self.fuel_web.get_last_created_cluster()
n_ctrls = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
cluster_id, ['controller'])
ctrl_ids = ",".join([str(nailgun_node['id'])
for nailgun_node in n_ctrls])
self.show_step(2)
cgroups = [
{"process": "mysqld",
"controller": "memory",
"limit": "memory.swappiness",
"value": 0},
{"process": "rabbitmq",
"controller": "blkio",
"limit": "blkio.weight",
"value": 500},
{"process": "rabbitmq",
"controller": "memory",
"limit": "memory.swappiness",
"value": 0},
{"process": "keystone",
"controller": "cpu",
"limit": "cpu.shares",
"value": 50},
]
self.apply_cgroups(cgroups, ctrl_ids)
for nailgun_node in n_ctrls:
self.check_cgroups_on_node(nailgun_node, cgroups)
self.show_step(3)
cgroups = [
{"process": "mysqld",
"controller": "memory",
"limit": "memory.swappiness",
"value": 10},
{"process": "rabbitmq",
"controller": "blkio",
"limit": "blkio.weight",
"value": 400},
{"process": "rabbitmq",
"controller": "memory",
"limit": "memory.swappiness",
"value": 60},
{"process": "keystone",
"controller": "cpu",
"limit": "cpu.shares",
"value": 70},
]
self.apply_cgroups(cgroups, ctrl_ids)
for nailgun_node in n_ctrls:
self.check_cgroups_on_node(nailgun_node, cgroups)
self.show_step(4)
cgroups = [
{"process": "mysqld",
"controller": "memory"},
{"process": "rabbitmq",
"controller": "blkio"},
{"process": "rabbitmq",
"controller": "memory"},
{"process": "keystone",
"controller": "cpu"},
]
self.apply_cgroups(cgroups, ctrl_ids)
for nailgun_node in n_ctrls:
self.check_cgroups_on_node(nailgun_node, cgroups)
@test(depends_on=[deploy_ha_cgroup],
groups=['apply_relative_cgroups_after_deploy'])
@log_snapshot_after_test
def apply_relative_cgroups_after_deploy(self):
"""Apply relative cgroups limits to services
Scenario:
1. Revert snapshot deploy_ha_cgroup
2. Configure and validate cgroups for mysqld, rabbitmq
and keystone with relative memory count
Duration 15m
"""
self.show_step(1)
self.env.revert_snapshot("deploy_ha_cgroup")
cluster_id = self.fuel_web.get_last_created_cluster()
n_ctrls = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
cluster_id, ['controller'])
ctrl_ids = ",".join([str(nailgun_node['id'])
for nailgun_node in n_ctrls])
self.show_step(2)
cgroups = [
{"process": "mysqld",
"controller": "memory",
"limit": "memory.swappiness",
"value": 0},
{"process": "mysqld",
"controller": "memory",
"limit": "memory.soft_limit_in_bytes",
"value": "%5,10,3000"},
{"process": "rabbitmq",
"controller": "blkio",
"limit": "blkio.weight",
"value": 500},
{"process": "rabbitmq",
"controller": "memory",
"limit": "memory.soft_limit_in_bytes",
"value": "%99,10,250"},
{"process": "keystone",
"controller": "cpu",
"limit": "cpu.shares",
"value": 50},
{"process": "keystone",
"controller": "memory",
"limit": "memory.soft_limit_in_bytes",
"value": "%1,250,2500"},
]
self.apply_cgroups(cgroups, ctrl_ids)
memory = float("".join(self.ssh_manager.execute(
n_ctrls[0]["ip"], "facter memorysize_mb")["stdout"]))
for cgroup in cgroups:
if cgroup["limit"] == "memory.soft_limit_in_bytes":
# pylint: disable=no-member
percent, min_mem, max_mem = cgroup["value"].split(",")
# pylint: enable=no-member
percent = int(percent.replace("%", "")) * memory / 100
min_mem, max_mem = int(min_mem), int(max_mem)
value = sorted((min_mem, percent, max_mem))[1]
cgroup["value"] = int(value * 1024 * 1024)
logger.info("New cgroups to verify: {}".format(cgroups))
for nailgun_node in n_ctrls:
self.check_cgroups_on_node(nailgun_node, cgroups)
@test(depends_on=[deploy_ha_cgroup],
groups=['apply_cgroups_reboot_node'])
@log_snapshot_after_test
def apply_cgroups_reboot_node(self):
"""Apply cgroups limits to services, reboot, verify
Scenario:
1. Revert snapshot deploy_ha_cgroup
2. Configure and validate cgroups for mysqld, rabbitmq
and keystone
3. Reboot controller
4. Validate cgroups for mysqld, rabbitmq and keystone
Duration 15m
"""
self.show_step(1)
self.env.revert_snapshot("deploy_ha_cgroup")
cluster_id = self.fuel_web.get_last_created_cluster()
n_ctrls = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
cluster_id, ['controller'])
ctrl_ids = ",".join([str(nailgun_node['id'])
for nailgun_node in n_ctrls])
self.show_step(2)
cgroups = [
{"process": "mysqld",
"controller": "memory",
"limit": "memory.swappiness",
"value": 0},
{"process": "rabbitmq",
"controller": "blkio",
"limit": "blkio.weight",
"value": 500},
{"process": "rabbitmq",
"controller": "memory",
"limit": "memory.swappiness",
"value": 0},
{"process": "keystone",
"controller": "cpu",
"limit": "cpu.shares",
"value": 50},
]
self.apply_cgroups(cgroups, ctrl_ids)
for nailgun_node in n_ctrls:
self.check_cgroups_on_node(nailgun_node, cgroups)
self.show_step(3)
target_controller = self.fuel_web.get_nailgun_primary_node(
self.fuel_web.get_devops_node_by_nailgun_node(n_ctrls[0]))
self.fuel_web.cold_restart_nodes([target_controller])
self.show_step(4)
self.check_cgroups_on_node(n_ctrls[0], cgroups)