486 lines
17 KiB
Python
486 lines
17 KiB
Python
# Copyright 2016 Mirantis, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
from __future__ import division
|
|
|
|
import re
|
|
import json
|
|
|
|
from proboscis import test
|
|
from proboscis import asserts
|
|
|
|
from fuelweb_test.helpers.decorators import log_snapshot_after_test
|
|
from fuelweb_test import settings
|
|
from fuelweb_test import logger
|
|
from fuelweb_test.tests.base_test_case import SetupEnvironment
|
|
from fuelweb_test.tests.base_test_case import TestBasic
|
|
from fuelweb_test.helpers import utils
|
|
|
|
|
|
@test(groups=["cgroup_ha"])
|
|
class TestCgroupHa(TestBasic):
|
|
"""Tests for verification deployment with enabled cgroup."""
|
|
|
|
@staticmethod
|
|
def generate_attributes(cgroups):
|
|
"""Generate cluster attributes structure from cgroups dicts."""
|
|
|
|
attributes = {}
|
|
for cgroup in cgroups:
|
|
if "limit" not in cgroup:
|
|
limit = {}
|
|
else:
|
|
limit = {cgroup["limit"]: cgroup["value"]}
|
|
attributes = utils.dict_merge(attributes, {
|
|
cgroup["process"]: {
|
|
"label": cgroup["process"],
|
|
"type": "text",
|
|
"value": {
|
|
cgroup["controller"]: limit
|
|
}
|
|
}
|
|
})
|
|
|
|
for cgroup in attributes.values():
|
|
cgroup["value"] = json.dumps(cgroup["value"])
|
|
|
|
return {"editable": {"cgroups": attributes}}
|
|
|
|
@staticmethod
|
|
def check_cgconfig_setup(config, process, controller,
|
|
limit=None, value=None):
|
|
"""Check /etc/cgconfig.conf contains properly configured cgroup."""
|
|
|
|
actual_limit = config[process][controller]
|
|
|
|
if limit is None and value is None:
|
|
asserts.assert_equal(actual_limit, {},
|
|
"Actual limit is not empty: {}"
|
|
.format(actual_limit))
|
|
else:
|
|
asserts.assert_equal(actual_limit[limit], value,
|
|
"Actual value limit is not as expected for "
|
|
"process {}, controller {}, limit {}, "
|
|
"expected value = {}, actual == {}"
|
|
.format(process, controller, limit, value,
|
|
actual_limit[limit]))
|
|
|
|
@staticmethod
|
|
def generate_lscgroups(cgroups):
|
|
"""Generate a list of lscgroups entities from cgroups dicts."""
|
|
|
|
cpu_controller = "cpu,cpuacct"
|
|
return ["{}:/{}".format(cpu_controller
|
|
if cgroup["controller"] in cpu_controller
|
|
else cgroup["controller"], cgroup["process"])
|
|
for cgroup in cgroups]
|
|
|
|
def apply_cgroups(self, cgroups, node_ids):
|
|
cluster_id = self.fuel_web.get_last_created_cluster()
|
|
|
|
self.fuel_web.client.update_cluster_attributes(
|
|
cluster_id, self.generate_attributes(cgroups))
|
|
task = self.fuel_web.client.put_deployment_tasks_for_cluster(
|
|
cluster_id=cluster_id,
|
|
data=["upload_configuration", "configuration_symlink",
|
|
"hiera", "cgroups"],
|
|
node_id=node_ids)
|
|
self.fuel_web.assert_task_success(task)
|
|
|
|
def get_cgroups_config(self, nailgun_node):
|
|
"""Get /etc/cgconfig.conf from node, transform it to json and loads
|
|
|
|
Before transformation:
|
|
group mysqld {
|
|
memory {
|
|
memory.swappiness = 0;
|
|
}
|
|
}
|
|
group keystone {
|
|
cpu {
|
|
cpu.shares = 70;
|
|
}
|
|
}
|
|
group rabbitmq {
|
|
blkio {
|
|
blkio.weight = 500;
|
|
}
|
|
memory {
|
|
memory.swappiness = 0;
|
|
}
|
|
}
|
|
|
|
After transformation:
|
|
{
|
|
"mysqld": {
|
|
"memory": {
|
|
"memory.swappiness": 0
|
|
}
|
|
},
|
|
"keystone": {
|
|
"cpu": {
|
|
"cpu.shares": 70
|
|
}
|
|
},
|
|
"rabbitmq": {
|
|
"blkio": {
|
|
"blkio.weight": 500
|
|
},
|
|
"memory": {
|
|
"memory.swappiness": 0
|
|
}
|
|
}
|
|
}
|
|
"""
|
|
|
|
cmd = "cat /etc/cgconfig.conf"
|
|
result = self.ssh_manager.execute(nailgun_node['ip'], cmd)["stdout"]
|
|
cgroups_config = "".join([line for line in result
|
|
if not line.startswith("#")])
|
|
|
|
cgroups_to_json = [
|
|
('group ', ''), # Remove group tag
|
|
(' {', ': {'), # Replace { -> : {
|
|
('}', '},'), # Replace } -> },
|
|
(';', ','), # Replace ; -> ,
|
|
(' = ', ': '), # Replace = -> :
|
|
('[a-z_]+\.{0,1}[a-z_]*', '"\g<0>"'), # Wrap all words with " "
|
|
# Words could contain period
|
|
('[\s\S]*', '{\g<0> }'), # Wrap whole string with {}
|
|
(',[ \t\r\n]+}', '}') # Clear trailing commas
|
|
]
|
|
|
|
for pattern, replace in cgroups_to_json:
|
|
cgroups_config = re.sub(pattern, replace, cgroups_config)
|
|
|
|
return json.loads(cgroups_config)
|
|
|
|
def check_cgroups_on_node(self, nailgun_node, cgroups):
|
|
"""Complex validation of cgroups on particular node."""
|
|
|
|
cgroups_config = self.get_cgroups_config(nailgun_node)
|
|
|
|
for cgroup in cgroups:
|
|
logger.info("Check cgroup config for {} {} on node {}"
|
|
.format(cgroup["process"], cgroup["controller"],
|
|
nailgun_node['fqdn']))
|
|
self.check_cgconfig_setup(config=cgroups_config, **cgroup)
|
|
|
|
for lscgroup in self.generate_lscgroups(cgroups):
|
|
check_group_cmd = 'sudo lscgroup | fgrep -q {}'
|
|
logger.info('Check {} group existence on controller node {}'
|
|
.format(lscgroup, nailgun_node['fqdn']))
|
|
self.ssh_manager.check_call(nailgun_node['ip'],
|
|
check_group_cmd.format(lscgroup))
|
|
|
|
for cgroup in cgroups:
|
|
check_rule_cmd = ("fgrep {} /etc/cgrules.conf | fgrep -q {}"
|
|
.format(cgroup["process"], cgroup["controller"]))
|
|
|
|
logger.info("Check cgrule {} {} on controller node {}"
|
|
.format(cgroup["process"], cgroup["controller"],
|
|
nailgun_node['fqdn']))
|
|
|
|
self.ssh_manager.check_call(nailgun_node['ip'], check_rule_cmd)
|
|
|
|
@test(depends_on=[SetupEnvironment.prepare_slaves_5],
|
|
groups=['deploy_ha_cgroup'])
|
|
@log_snapshot_after_test
|
|
def deploy_ha_cgroup(self):
|
|
"""Deploy cluster in HA mode with enabled cgroups
|
|
|
|
Scenario:
|
|
1. Create cluster
|
|
2. Add 3 nodes with controller role
|
|
3. Add 1 node with compute role
|
|
4. Add 1 node with cinder role
|
|
5. Deploy the cluster
|
|
6. Check ceph status
|
|
7. Run OSTF
|
|
|
|
Duration 90m
|
|
Snapshot deploy_ha_cgroup
|
|
"""
|
|
self.check_run("deploy_ha_cgroup")
|
|
self.env.revert_snapshot("ready_with_5_slaves")
|
|
data = {
|
|
'tenant': 'cgroup',
|
|
'user': 'cgroup',
|
|
'password': 'cgroup',
|
|
'net_provider': 'neutron',
|
|
'net_segment_type': settings.NEUTRON_SEGMENT['vlan']
|
|
}
|
|
|
|
cluster_id = self.fuel_web.create_cluster(
|
|
name=self.__class__.__name__,
|
|
mode=settings.DEPLOYMENT_MODE,
|
|
settings=data)
|
|
|
|
self.fuel_web.update_nodes(
|
|
cluster_id,
|
|
{
|
|
'slave-01': ['controller'],
|
|
'slave-02': ['controller'],
|
|
'slave-03': ['controller'],
|
|
'slave-04': ['compute'],
|
|
'slave-05': ['cinder']
|
|
}
|
|
)
|
|
|
|
cgroup_data = [{
|
|
"process": "keystone",
|
|
"controller": "cpu",
|
|
"limit": "cpu.shares",
|
|
"value": 70,
|
|
}]
|
|
|
|
self.fuel_web.client.update_cluster_attributes(
|
|
cluster_id, self.generate_attributes(cgroup_data))
|
|
|
|
# Cluster deploy
|
|
self.fuel_web.deploy_cluster_wait(cluster_id)
|
|
|
|
# Run ostf
|
|
self.fuel_web.run_ostf(cluster_id=cluster_id)
|
|
|
|
# Check that task cgroup was executed
|
|
cmd = 'fgrep "MODULAR: cgroups/cgroups.pp" -q /var/log/puppet.log'
|
|
n_ctrls = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
|
|
cluster_id, ['controller'])
|
|
for nailgun_node in n_ctrls:
|
|
logger.info('Check cgroups task on controller node {0}'.format(
|
|
nailgun_node["fqdn"]))
|
|
|
|
self.ssh_manager.check_call(nailgun_node['ip'], cmd)
|
|
|
|
self.check_cgroups_on_node(nailgun_node, cgroup_data)
|
|
|
|
self.env.make_snapshot("deploy_ha_cgroup", is_make=True)
|
|
|
|
@test(depends_on=[deploy_ha_cgroup],
|
|
groups=['apply_cgroups_after_deploy'])
|
|
@log_snapshot_after_test
|
|
def apply_cgroups_after_deploy(self):
|
|
"""Apply, reconfigure and disable cgroups limits to services
|
|
|
|
Scenario:
|
|
1. Revert snapshot deploy_ha_cgroup
|
|
2. Configure and validate cgroups for mysqld, rabbitmq
|
|
and keystone
|
|
3. Reconfigure and validate cgroups for mysqld,
|
|
rabbitmq and keystone
|
|
4. Disable cgroups for mysqld, rabbitmq and keystone
|
|
|
|
Duration 15m
|
|
"""
|
|
|
|
self.show_step(1)
|
|
self.env.revert_snapshot("deploy_ha_cgroup")
|
|
|
|
cluster_id = self.fuel_web.get_last_created_cluster()
|
|
n_ctrls = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
|
|
cluster_id, ['controller'])
|
|
ctrl_ids = ",".join([str(nailgun_node['id'])
|
|
for nailgun_node in n_ctrls])
|
|
|
|
self.show_step(2)
|
|
cgroups = [
|
|
{"process": "mysqld",
|
|
"controller": "memory",
|
|
"limit": "memory.swappiness",
|
|
"value": 0},
|
|
{"process": "rabbitmq",
|
|
"controller": "blkio",
|
|
"limit": "blkio.weight",
|
|
"value": 500},
|
|
{"process": "rabbitmq",
|
|
"controller": "memory",
|
|
"limit": "memory.swappiness",
|
|
"value": 0},
|
|
{"process": "keystone",
|
|
"controller": "cpu",
|
|
"limit": "cpu.shares",
|
|
"value": 50},
|
|
]
|
|
|
|
self.apply_cgroups(cgroups, ctrl_ids)
|
|
for nailgun_node in n_ctrls:
|
|
self.check_cgroups_on_node(nailgun_node, cgroups)
|
|
|
|
self.show_step(3)
|
|
cgroups = [
|
|
{"process": "mysqld",
|
|
"controller": "memory",
|
|
"limit": "memory.swappiness",
|
|
"value": 10},
|
|
{"process": "rabbitmq",
|
|
"controller": "blkio",
|
|
"limit": "blkio.weight",
|
|
"value": 400},
|
|
{"process": "rabbitmq",
|
|
"controller": "memory",
|
|
"limit": "memory.swappiness",
|
|
"value": 60},
|
|
{"process": "keystone",
|
|
"controller": "cpu",
|
|
"limit": "cpu.shares",
|
|
"value": 70},
|
|
]
|
|
|
|
self.apply_cgroups(cgroups, ctrl_ids)
|
|
for nailgun_node in n_ctrls:
|
|
self.check_cgroups_on_node(nailgun_node, cgroups)
|
|
|
|
self.show_step(4)
|
|
cgroups = [
|
|
{"process": "mysqld",
|
|
"controller": "memory"},
|
|
{"process": "rabbitmq",
|
|
"controller": "blkio"},
|
|
{"process": "rabbitmq",
|
|
"controller": "memory"},
|
|
{"process": "keystone",
|
|
"controller": "cpu"},
|
|
]
|
|
|
|
self.apply_cgroups(cgroups, ctrl_ids)
|
|
for nailgun_node in n_ctrls:
|
|
self.check_cgroups_on_node(nailgun_node, cgroups)
|
|
|
|
@test(depends_on=[deploy_ha_cgroup],
|
|
groups=['apply_relative_cgroups_after_deploy'])
|
|
@log_snapshot_after_test
|
|
def apply_relative_cgroups_after_deploy(self):
|
|
"""Apply relative cgroups limits to services
|
|
|
|
Scenario:
|
|
1. Revert snapshot deploy_ha_cgroup
|
|
2. Configure and validate cgroups for mysqld, rabbitmq
|
|
and keystone with relative memory count
|
|
|
|
Duration 15m
|
|
"""
|
|
self.show_step(1)
|
|
self.env.revert_snapshot("deploy_ha_cgroup")
|
|
|
|
cluster_id = self.fuel_web.get_last_created_cluster()
|
|
n_ctrls = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
|
|
cluster_id, ['controller'])
|
|
ctrl_ids = ",".join([str(nailgun_node['id'])
|
|
for nailgun_node in n_ctrls])
|
|
|
|
self.show_step(2)
|
|
cgroups = [
|
|
{"process": "mysqld",
|
|
"controller": "memory",
|
|
"limit": "memory.swappiness",
|
|
"value": 0},
|
|
{"process": "mysqld",
|
|
"controller": "memory",
|
|
"limit": "memory.soft_limit_in_bytes",
|
|
"value": "%5,10,3000"},
|
|
{"process": "rabbitmq",
|
|
"controller": "blkio",
|
|
"limit": "blkio.weight",
|
|
"value": 500},
|
|
{"process": "rabbitmq",
|
|
"controller": "memory",
|
|
"limit": "memory.soft_limit_in_bytes",
|
|
"value": "%99,10,250"},
|
|
{"process": "keystone",
|
|
"controller": "cpu",
|
|
"limit": "cpu.shares",
|
|
"value": 50},
|
|
{"process": "keystone",
|
|
"controller": "memory",
|
|
"limit": "memory.soft_limit_in_bytes",
|
|
"value": "%1,250,2500"},
|
|
]
|
|
|
|
self.apply_cgroups(cgroups, ctrl_ids)
|
|
|
|
memory = float("".join(self.ssh_manager.execute(
|
|
n_ctrls[0]["ip"], "facter memorysize_mb")["stdout"]))
|
|
|
|
for cgroup in cgroups:
|
|
if cgroup["limit"] == "memory.soft_limit_in_bytes":
|
|
# pylint: disable=no-member
|
|
percent, min_mem, max_mem = cgroup["value"].split(",")
|
|
# pylint: enable=no-member
|
|
percent = int(percent.replace("%", "")) * memory / 100
|
|
min_mem, max_mem = int(min_mem), int(max_mem)
|
|
|
|
value = sorted((min_mem, percent, max_mem))[1]
|
|
cgroup["value"] = int(value * 1024 * 1024)
|
|
|
|
logger.info("New cgroups to verify: {}".format(cgroups))
|
|
for nailgun_node in n_ctrls:
|
|
self.check_cgroups_on_node(nailgun_node, cgroups)
|
|
|
|
@test(depends_on=[deploy_ha_cgroup],
|
|
groups=['apply_cgroups_reboot_node'])
|
|
@log_snapshot_after_test
|
|
def apply_cgroups_reboot_node(self):
|
|
"""Apply cgroups limits to services, reboot, verify
|
|
|
|
Scenario:
|
|
1. Revert snapshot deploy_ha_cgroup
|
|
2. Configure and validate cgroups for mysqld, rabbitmq
|
|
and keystone
|
|
3. Reboot controller
|
|
4. Validate cgroups for mysqld, rabbitmq and keystone
|
|
|
|
Duration 15m
|
|
"""
|
|
|
|
self.show_step(1)
|
|
self.env.revert_snapshot("deploy_ha_cgroup")
|
|
|
|
cluster_id = self.fuel_web.get_last_created_cluster()
|
|
n_ctrls = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
|
|
cluster_id, ['controller'])
|
|
ctrl_ids = ",".join([str(nailgun_node['id'])
|
|
for nailgun_node in n_ctrls])
|
|
|
|
self.show_step(2)
|
|
cgroups = [
|
|
{"process": "mysqld",
|
|
"controller": "memory",
|
|
"limit": "memory.swappiness",
|
|
"value": 0},
|
|
{"process": "rabbitmq",
|
|
"controller": "blkio",
|
|
"limit": "blkio.weight",
|
|
"value": 500},
|
|
{"process": "rabbitmq",
|
|
"controller": "memory",
|
|
"limit": "memory.swappiness",
|
|
"value": 0},
|
|
{"process": "keystone",
|
|
"controller": "cpu",
|
|
"limit": "cpu.shares",
|
|
"value": 50},
|
|
]
|
|
|
|
self.apply_cgroups(cgroups, ctrl_ids)
|
|
for nailgun_node in n_ctrls:
|
|
self.check_cgroups_on_node(nailgun_node, cgroups)
|
|
|
|
self.show_step(3)
|
|
target_controller = self.fuel_web.get_nailgun_primary_node(
|
|
self.fuel_web.get_devops_node_by_nailgun_node(n_ctrls[0]))
|
|
self.fuel_web.cold_restart_nodes([target_controller])
|
|
|
|
self.show_step(4)
|
|
self.check_cgroups_on_node(n_ctrls[0], cgroups)
|