78109fed9e
- Correct custom group name of the test (as it was refactored to use template based runner) to have the existing CI job being able to pick and run the moved test. - Remove the inital version of the test. Change-Id: If5bde759d39f2779ea0c9e4b74c1ab94b91774ff Partial-bug: #1512677
443 lines
16 KiB
Python
443 lines
16 KiB
Python
# Copyright 2013 Mirantis, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import datetime
|
|
import xmlrpclib
|
|
|
|
from devops.helpers.helpers import http
|
|
from devops.helpers.helpers import wait
|
|
from proboscis.asserts import assert_equal
|
|
from proboscis.asserts import assert_true
|
|
from proboscis import test
|
|
|
|
from fuelweb_test.helpers.decorators import log_snapshot_after_test
|
|
from fuelweb_test import logger
|
|
from fuelweb_test.tests.base_test_case import SetupEnvironment
|
|
from fuelweb_test.tests.base_test_case import TestBasic
|
|
|
|
|
|
@test(groups=["thread_1"])
|
|
class TestAdminNode(TestBasic):
|
|
"""TestAdminNode.""" # TODO documentation
|
|
|
|
@test(depends_on=[SetupEnvironment.setup_master],
|
|
groups=["test_cobbler_alive"])
|
|
@log_snapshot_after_test
|
|
def test_cobbler_alive(self):
|
|
"""Test current installation has correctly setup cobbler
|
|
|
|
API and cobbler HTTP server are alive
|
|
|
|
Scenario:
|
|
1. Revert snapshot "empty"
|
|
2. test cobbler API and HTTP server through send http request
|
|
|
|
Duration 1m
|
|
|
|
"""
|
|
self.env.revert_snapshot("empty")
|
|
wait(
|
|
lambda: http(host=self.env.get_admin_node_ip(), url='/cobbler_api',
|
|
waited_code=501),
|
|
timeout=60
|
|
)
|
|
server = xmlrpclib.Server(
|
|
'http://%s/cobbler_api' % self.env.get_admin_node_ip())
|
|
|
|
config = self.env.admin_actions.get_fuel_settings()
|
|
username = config['cobbler']['user']
|
|
password = config['cobbler']['password']
|
|
|
|
# raises an error if something isn't right
|
|
server.login(username, password)
|
|
|
|
@test(depends_on=[SetupEnvironment.setup_master],
|
|
groups=["test_astuted_alive"])
|
|
@log_snapshot_after_test
|
|
def test_astuted_alive(self):
|
|
"""Test astute master and worker processes are alive on master node
|
|
|
|
Scenario:
|
|
1. Revert snapshot "empty"
|
|
2. Search for master and child processes
|
|
|
|
Duration 1m
|
|
|
|
"""
|
|
self.env.revert_snapshot("empty")
|
|
with self.env.d_env.get_admin_remote() as remote:
|
|
ps_output = remote.execute('ps ax')['stdout']
|
|
astute_master = filter(lambda x: 'astute master' in x, ps_output)
|
|
logger.info("Found astute processes: %s" % astute_master)
|
|
assert_equal(len(astute_master), 1)
|
|
astute_workers = filter(lambda x: 'astute worker' in x, ps_output)
|
|
logger.info(
|
|
"Found %d astute worker processes: %s" %
|
|
(len(astute_workers), astute_workers))
|
|
assert_equal(True, len(astute_workers) > 1)
|
|
|
|
|
|
@test(groups=["logrotate"])
|
|
class TestLogrotateBase(TestBasic):
|
|
|
|
def generate_file(self, remote, name, path, size):
|
|
cmd = 'cd {0} && fallocate -l {1} {2}'.format(path, size, name)
|
|
result = remote.execute(cmd)
|
|
assert_equal(0, result['exit_code'],
|
|
'Command {0} execution failed. '
|
|
'Execution result is: {1}'.format(cmd, result))
|
|
|
|
def execute_logrotate_cmd(self, remote, cmd=None, exit_code=None):
|
|
if not cmd:
|
|
cmd = 'logrotate -v -f /etc/logrotate.conf'
|
|
result = remote.execute(cmd)
|
|
logger.debug(
|
|
'Results of command {0} execution exit_code:{1} '
|
|
'stdout: {2} stderr: {3}'.format(
|
|
cmd, result['exit_code'], result['stdout'], result['stderr']))
|
|
if not exit_code:
|
|
assert_equal(0, result['exit_code'],
|
|
'Command {0} execution failed. '
|
|
'Execution result is: {1}'.format(cmd, result))
|
|
else:
|
|
return result
|
|
|
|
def check_free_space(self, remote, return_as_is=None):
|
|
result = remote.execute(
|
|
'python -c "import os; '
|
|
'stats=os.statvfs(\'/var/log\'); '
|
|
'print stats.f_bavail * stats.f_frsize"')
|
|
assert_equal(0, result['exit_code'],
|
|
'Failed to check free '
|
|
'space with {0}'. format(result))
|
|
if not return_as_is:
|
|
return self.bytestogb(int(result['stdout'][0]))
|
|
else:
|
|
return int(result['stdout'][0])
|
|
|
|
def check_free_inodes(self, remote):
|
|
result = remote.execute(
|
|
'python -c "import os; '
|
|
'stats=os.statvfs(\'/var/log\'); '
|
|
'print stats.f_ffree"')
|
|
assert_equal(0, result['exit_code'],
|
|
'Failed to check free '
|
|
'inodes with {0}'. format(result))
|
|
return self.bytestogb(int(result['stdout'][0]))
|
|
|
|
def bytestogb(self, data):
|
|
symbols = ('K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')
|
|
prefix = {}
|
|
for i, s in enumerate(symbols):
|
|
prefix[s] = 1 << (i + 1) * 10
|
|
for s in reversed(symbols):
|
|
if data >= prefix[s]:
|
|
value = float(data) / prefix[s]
|
|
return format(value, '.1f'), s
|
|
return data, 'B'
|
|
|
|
def create_old_file(self, remote, name):
|
|
one_week_old = datetime.datetime.now() - datetime.timedelta(days=7)
|
|
res = remote.execute(
|
|
'touch {0} -d {1}'.format(name, one_week_old))
|
|
assert_equal(0, res['exit_code'],
|
|
'Failed to create old '
|
|
'file with next result: {0}'.format(res))
|
|
return res
|
|
|
|
@test(depends_on=[SetupEnvironment.setup_master],
|
|
groups=["test_logrotate"])
|
|
@log_snapshot_after_test
|
|
def test_log_rotation(self):
|
|
"""Logrotate with logrotate.conf on master node
|
|
|
|
Scenario:
|
|
1. Revert snapshot "empty"
|
|
2. Check free disk space under /var/log, check free inodes
|
|
3. Generate 2GB size file
|
|
4. Run logrotate 2 times
|
|
5. Check free disk space, check free inodes
|
|
|
|
Duration 30m
|
|
|
|
"""
|
|
self.env.revert_snapshot("empty")
|
|
with self.env.d_env.get_admin_remote() as remote:
|
|
|
|
# get data before logrotate
|
|
free, suff = self.check_free_space(remote)
|
|
|
|
free_inodes, i_suff = self.check_free_inodes(remote)
|
|
logger.debug('Free inodes before file '
|
|
'creation: {0}{1}'.format(free_inodes, i_suff))
|
|
|
|
self.generate_file(
|
|
remote, size='2G',
|
|
path='/var/log/',
|
|
name='messages')
|
|
|
|
free2, suff2 = self.check_free_space(remote)
|
|
assert_true(
|
|
free2 < free,
|
|
'File was not created. Free space '
|
|
'before creation {0}{1}, '
|
|
'free space after '
|
|
'creation {2}{3}'.format(free, suff, free2, suff2))
|
|
|
|
self.execute_logrotate_cmd(remote)
|
|
|
|
free3, suff3 = self.check_free_space(remote)
|
|
res = self.execute_logrotate_cmd(remote, exit_code=1)
|
|
|
|
# Expect 1 exit code here, according
|
|
# to some rotated logs are skipped to rotate
|
|
# second run. That's caused 1
|
|
assert_equal(1, res['exit_code'])
|
|
assert_equal(
|
|
False, 'error' in res['stderr'],
|
|
'Second run of logrotate failed'
|
|
' with {0}'.format(res['stderr']))
|
|
|
|
free4, suff4 = self.check_free_space(remote)
|
|
free_inodes4, i_suff4 = self.check_free_inodes(remote)
|
|
logger.info('Free inodes after logrotation:'
|
|
' {0}{1}'.format(free_inodes4, i_suff4))
|
|
|
|
assert_true(
|
|
free4 > free3,
|
|
'Logs were not rotated. '
|
|
'Rotate was executed 2 times. '
|
|
'Free space after first rotation: {0}{1}, '
|
|
'after second {2}{3} free space before rotation {4}'
|
|
'{5}'.format(free3, suff3, free4, suff4, free, suff))
|
|
|
|
assert_equal(
|
|
(free_inodes, i_suff),
|
|
(free_inodes4, i_suff4),
|
|
'Unexpected free inodes count. Before log rotate was: {0}{1}'
|
|
' after logrotation: {2}{3}'.format(
|
|
free_inodes, i_suff, free_inodes4, i_suff4))
|
|
self.env.make_snapshot("test_logrotate")
|
|
|
|
@test(depends_on=[SetupEnvironment.setup_master],
|
|
groups=["test_fuel_nondaily_logrotate"])
|
|
@log_snapshot_after_test
|
|
def test_fuel_nondaily_rotation(self):
|
|
"""Logrotate with fuel.nondaily on master node
|
|
|
|
Scenario:
|
|
1. Revert snapshot "empty"
|
|
2. Check free disk space under /var/log, check free inodes
|
|
3. Generate 2GB /var/log/ostf-test.log size file
|
|
4. Run /usr/bin/fuel-logrotate
|
|
5. Check free disk space, check free inodes
|
|
|
|
Duration 30m
|
|
|
|
"""
|
|
self.env.revert_snapshot("empty")
|
|
with self.env.d_env.get_admin_remote() as remote:
|
|
|
|
# get data before logrotate
|
|
free, suff = self.check_free_space(remote)
|
|
free_inodes, i_suff = self.check_free_inodes(remote)
|
|
logger.debug('Free inodes before file '
|
|
'creation: {0}{1}'.format(free_inodes, i_suff))
|
|
|
|
self.generate_file(
|
|
remote, size='2G',
|
|
path='/var/log/',
|
|
name='ostf-test.log')
|
|
|
|
free2, suff2 = self.check_free_space(remote)
|
|
assert_true(
|
|
free2 < free,
|
|
'File was not created. Free space '
|
|
'before creation {0}{1}, '
|
|
'free space after '
|
|
'creation {2}{3}'.format(free, suff, free2, suff2))
|
|
|
|
self.execute_logrotate_cmd(remote, cmd='/usr/bin/fuel-logrotate')
|
|
|
|
free3, suff3 = self.check_free_space(remote)
|
|
free_inodes3, i_suff3 = self.check_free_inodes(remote)
|
|
logger.info('Free inodes after logrotation:'
|
|
' {0}{1}'.format(free_inodes3, i_suff3))
|
|
|
|
assert_true(
|
|
free3 > free2,
|
|
'Logs were not rotated. '
|
|
'Free space before rotation: {0}{1}, '
|
|
'after rotation {2}{3}'.format(free2, suff2, free3, suff3))
|
|
|
|
assert_equal(
|
|
(free_inodes, i_suff),
|
|
(free_inodes3, i_suff3),
|
|
'Unexpected free inodes count. Before log rotate was: {0}{1}'
|
|
' after logrotation: {2}{3}'.format(
|
|
free_inodes, i_suff, free_inodes3, i_suff3))
|
|
|
|
self.env.make_snapshot("test_fuel_nondaily_logrotate")
|
|
|
|
@test(depends_on=[SetupEnvironment.setup_master],
|
|
groups=["test_logrotate_101MB"])
|
|
@log_snapshot_after_test
|
|
def test_log_rotation_101MB(self):
|
|
"""Logrotate with logrotate.conf for 101MB size file on master node
|
|
|
|
Scenario:
|
|
1. Revert snapshot "empty"
|
|
2. Check free disk space and free inodes under /var/log
|
|
3. Generate 101MB size file
|
|
4. Run logrotate 2 times
|
|
5. Check free disk space and free inodes
|
|
|
|
Duration 30m
|
|
|
|
"""
|
|
self.env.revert_snapshot("empty")
|
|
with self.env.d_env.get_admin_remote() as remote:
|
|
|
|
# get data before logrotate
|
|
free, suff = self.check_free_space(remote)
|
|
|
|
free_inodes, i_suff = self.check_free_inodes(remote)
|
|
logger.debug('Free inodes before file '
|
|
'creation: {0}{1}'.format(free_inodes, i_suff))
|
|
|
|
self.generate_file(
|
|
remote, size='101M',
|
|
path='/var/log/',
|
|
name='messages')
|
|
|
|
free2, suff2 = self.check_free_space(remote)
|
|
assert_true(
|
|
free2 < free,
|
|
'File was not created. Free space '
|
|
'before creation {0}{1}, '
|
|
'free space after '
|
|
'creation {2}{3}'.format(free, suff, free2, suff2))
|
|
|
|
self.execute_logrotate_cmd(remote)
|
|
|
|
free3, suff3 = self.check_free_space(remote)
|
|
res = self.execute_logrotate_cmd(remote, exit_code=1)
|
|
|
|
# Expect 1 exit code here, according
|
|
# to some rotated logs are skipped to rotate
|
|
# second run. That's caused 1
|
|
assert_equal(1, res['exit_code'])
|
|
assert_equal(
|
|
False, 'error' in res['stderr'],
|
|
'Second run of logrotate failed'
|
|
' with {0}'.format(res['stderr']))
|
|
|
|
free4, suff4 = self.check_free_space(remote)
|
|
free_inodes4, i_suff4 = self.check_free_inodes(remote)
|
|
logger.info('Free inodes after logrotation:'
|
|
' {0}{1}'.format(free_inodes4, i_suff4))
|
|
|
|
assert_true(
|
|
free4 > free3,
|
|
'Logs were not rotated. '
|
|
'Rotate was executed 2 times. '
|
|
'Free space after first rotation: {0}{1}, '
|
|
'after second {2}{3} free space before rotation {4}'
|
|
'{5}'.format(free3, suff3, free4, suff4, free, suff))
|
|
|
|
assert_equal(
|
|
(free_inodes, i_suff),
|
|
(free_inodes4, i_suff4),
|
|
'Unexpected free inodes count. Before log rotate was: {0}{1}'
|
|
' after logrotation: {2}{3}'.format(
|
|
free_inodes, i_suff, free_inodes4, i_suff4))
|
|
self.env.make_snapshot("test_logrotate_101MB")
|
|
|
|
@test(depends_on=[SetupEnvironment.setup_master],
|
|
groups=["test_logrotate_one_week_11MB"])
|
|
@log_snapshot_after_test
|
|
def test_log_rotation_one_week_11MB(self):
|
|
"""Logrotate with logrotate.conf for 1 week old file with size 11MB
|
|
|
|
Scenario:
|
|
1. Revert snapshot "empty"
|
|
2. Check free disk space and free inodes under /var/log
|
|
3. Generate 1 week old 11MB size file
|
|
4. Run logrotate 2 times
|
|
5. Check free disk space and free inodes
|
|
|
|
Duration 30m
|
|
|
|
"""
|
|
self.env.revert_snapshot("empty")
|
|
with self.env.d_env.get_admin_remote() as remote:
|
|
|
|
# get data before logrotate
|
|
free = self.check_free_space(remote, return_as_is=True)
|
|
|
|
free_inodes, i_suff = self.check_free_inodes(remote)
|
|
logger.debug('Free inodes before file '
|
|
'creation: {0}{1}'.format(free_inodes, i_suff))
|
|
# create 1 week old empty file
|
|
|
|
self.create_old_file(remote, name='/var/log/messages')
|
|
|
|
self.generate_file(
|
|
remote, size='11M',
|
|
path='/var/log/',
|
|
name='messages')
|
|
|
|
free2 = self.check_free_space(remote, return_as_is=True)
|
|
assert_true(
|
|
free2 < free,
|
|
'File was not created. Free space '
|
|
'before creation {0}, '
|
|
'free space after '
|
|
'creation {1}'.format(free, free2))
|
|
|
|
self.execute_logrotate_cmd(remote)
|
|
|
|
free3 = self.check_free_space(remote, return_as_is=True)
|
|
res = self.execute_logrotate_cmd(remote, exit_code=1)
|
|
|
|
# Expect 1 exit code here, according
|
|
# to some rotated logs are skipped to rotate
|
|
# second run. That's caused 1
|
|
assert_equal(1, res['exit_code'])
|
|
assert_equal(
|
|
False, 'error' in res['stderr'],
|
|
'Second run of logrotate failed'
|
|
' with {0}'.format(res['stderr']))
|
|
|
|
free4 = self.check_free_space(remote, return_as_is=True)
|
|
free_inodes4, i_suff4 = self.check_free_inodes(remote)
|
|
logger.info('Free inodes after logrotation:'
|
|
' {0}{1}'.format(free_inodes4, i_suff4))
|
|
|
|
assert_true(
|
|
free4 > free3,
|
|
'Logs were not rotated. '
|
|
'Rotate was executed 2 times. '
|
|
'Free space after first rotation: {0}, '
|
|
'after second {1} free space before rotation'
|
|
'{2}'.format(free3, free4, free))
|
|
|
|
assert_equal(
|
|
(free_inodes, i_suff),
|
|
(free_inodes4, i_suff4),
|
|
'Unexpected free inodes count. Before log rotate was: {0}{1}'
|
|
' after logrotation: {2}{3}'.format(
|
|
free_inodes, i_suff, free_inodes4, i_suff4))
|
|
self.env.make_snapshot("test_logrotate_one_week_11MB")
|