openstack
/
validations-libs
Code Issues Proposed changes

Merge "Improved log path handling"

This commit is contained in:
Zuul 2021-06-06 18:48:07 +00:00 committed by Gerrit Code Review
parent 858d071820 f2830c5244
commit 28db579748
5 changed files with 101 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
validations_libs/constants.py
View File

@ -20,9 +20,7 @@ ANSIBLE_VALIDATION_DIR = '/usr/share/ansible/validation-playbooks'
VALIDATION_GROUPS_INFO = '%s/groups.yaml' % DEFAULT_VALIDATIONS_BASEDIR VALIDATION_GROUPS_INFO = '%s/groups.yaml' % DEFAULT_VALIDATIONS_BASEDIR
VALIDATIONS_LOG_BASEDIR = ('/var/log/validations' VALIDATIONS_LOG_BASEDIR = os.path.join(os.environ.get('HOME'), 'validations')
if os.path.exists('/var/log/validations') else
os.getcwd())
VALIDATION_ANSIBLE_ARTIFACT_PATH = '{}/artifacts/'.format( VALIDATION_ANSIBLE_ARTIFACT_PATH = '{}/artifacts/'.format(
VALIDATIONS_LOG_BASEDIR) VALIDATIONS_LOG_BASEDIR)

12
validations_libs/tests/test_utils.py
View File

@ -47,18 +47,6 @@ class TestUtils(TestCase):
utils.get_validations_data, utils.get_validations_data,
validation) validation)
@mock.patch('validations_libs.utils.current_time',
return_value='2020-04-02T06:58:20.352272Z')
@mock.patch('os.makedirs')
@mock.patch('uuid.uuid4', return_value='1234')
def test_create_artifacts_dir(self, mock_uuid, mock_makedirs,
mock_datetime):
uuid, dir_path = utils.create_artifacts_dir(dir_path='/tmp/foo',
prefix='ntp')
self.assertEqual(uuid, '1234')
self.assertEqual(dir_path,
'/tmp/foo/1234_ntp_2020-04-02T06:58:20.352272Z')
@mock.patch('yaml.safe_load', return_value=fakes.FAKE_PLAYBOOK) @mock.patch('yaml.safe_load', return_value=fakes.FAKE_PLAYBOOK)
@mock.patch('six.moves.builtins.open') @mock.patch('six.moves.builtins.open')
@mock.patch('glob.glob') @mock.patch('glob.glob')

100
validations_libs/tests/test_validation_actions.py
View File

@ -45,11 +45,9 @@ class TestValidationActions(TestCase):
'My Validation Two Name', 'My Validation Two Name',
['prep', 'pre-introspection'])])) ['prep', 'pre-introspection'])]))
@mock.patch('validations_libs.utils.create_artifacts_dir',
return_value=('1234', '/tmp/'))
@mock.patch('validations_libs.utils.get_validations_playbook', @mock.patch('validations_libs.utils.get_validations_playbook',
return_value=['/tmp/foo/fake.yaml']) return_value=['/tmp/foo/fake.yaml'])
def test_validation_skip_validation(self, mock_validation_play, mock_tmp): def test_validation_skip_validation(self, mock_validation_play):
playbook = ['fake.yaml'] playbook = ['fake.yaml']
inventory = 'tmp/inventory.yaml' inventory = 'tmp/inventory.yaml'
@ -66,16 +64,29 @@ class TestValidationActions(TestCase):
limit_hosts=None) limit_hosts=None)
self.assertEqual(run_return, []) self.assertEqual(run_return, [])
@mock.patch('validations_libs.utils.current_time',
return_value='time')
@mock.patch('validations_libs.utils.uuid.uuid4',
return_value='123')
@mock.patch('validations_libs.utils.os.makedirs')
@mock.patch('validations_libs.utils.os.access',
return_value=True)
@mock.patch('validations_libs.utils.os.path.exists',
return_value=True)
@mock.patch('validations_libs.utils.get_validations_playbook', @mock.patch('validations_libs.utils.get_validations_playbook',
return_value=['/tmp/foo/fake.yaml']) return_value=['/tmp/foo/fake.yaml'])
@mock.patch('validations_libs.ansible.Ansible.run') @mock.patch('validations_libs.ansible.Ansible.run')
@mock.patch('validations_libs.utils.create_artifacts_dir', def test_validation_skip_on_specific_host(self, mock_ansible_run,
return_value=('1234', '/tmp/')) mock_validation_play,
def test_validation_skip_on_specific_host(self, mock_tmp, mock_ansible_run, mock_exists,
mock_validation_play): mock_access,
mock_makedirs,
mock_uuid,
mock_time):
mock_ansible_run.return_value = ('fake.yaml', 0, 'successful') mock_ansible_run.return_value = ('fake.yaml', 0, 'successful')
run_called_args = { run_called_args = {
'workdir': '/tmp/', 'workdir': '/var/log/validations/artifacts/123_fake.yaml_time',
'playbook': '/tmp/foo/fake.yaml', 'playbook': '/tmp/foo/fake.yaml',
'base_dir': '/usr/share/ansible/', 'base_dir': '/usr/share/ansible/',
'playbook_dir': '/tmp/foo', 'playbook_dir': '/tmp/foo',
@ -86,11 +97,11 @@ class TestValidationActions(TestCase):
'quiet': True, 'quiet': True,
'extra_vars': None, 'extra_vars': None,
'limit_hosts': '!cloud1', 'limit_hosts': '!cloud1',
'ansible_artifact_path': '/tmp/',
'extra_env_variables': None, 'extra_env_variables': None,
'ansible_cfg': None, 'ansible_cfg': None,
'gathering_policy': 'explicit', 'gathering_policy': 'explicit',
'log_path': None, 'ansible_artifact_path': '/var/log/validations/artifacts/123_fake.yaml_time',
'log_path': '/var/log/validations',
'run_async': False, 'run_async': False,
'python_interpreter': None, 'python_interpreter': None,
'ssh_user': None 'ssh_user': None
@ -106,22 +117,36 @@ class TestValidationActions(TestCase):
run = ValidationActions() run = ValidationActions()
run_return = run.run_validations(playbook, inventory, run_return = run.run_validations(playbook, inventory,
log_path='/var/log/validations',
validations_dir='/tmp/foo', validations_dir='/tmp/foo',
skip_list=skip_list, skip_list=skip_list,
limit_hosts='!cloud1', limit_hosts='!cloud1')
)
mock_ansible_run.assert_called_with(**run_called_args) mock_ansible_run.assert_called_with(**run_called_args)
@mock.patch('validations_libs.utils.current_time',
return_value='time')
@mock.patch('validations_libs.utils.uuid.uuid4',
return_value='123')
@mock.patch('validations_libs.utils.os.makedirs')
@mock.patch('validations_libs.utils.os.access',
return_value=True)
@mock.patch('validations_libs.utils.os.path.exists',
return_value=True)
@mock.patch('validations_libs.utils.get_validations_playbook', @mock.patch('validations_libs.utils.get_validations_playbook',
return_value=['/tmp/foo/fake.yaml']) return_value=['/tmp/foo/fake.yaml'])
@mock.patch('validations_libs.ansible.Ansible.run') @mock.patch('validations_libs.ansible.Ansible.run')
@mock.patch('validations_libs.utils.create_artifacts_dir', def test_validation_skip_with_limit_host(self, mock_ansible_run,
return_value=('1234', '/tmp/')) mock_validation_play,
def test_validation_skip_with_limit_host(self, mock_tmp, mock_ansible_run, mock_exists,
mock_validation_play): mock_access,
mock_makedirs,
mock_uuid,
mock_time):
mock_ansible_run.return_value = ('fake.yaml', 0, 'successful') mock_ansible_run.return_value = ('fake.yaml', 0, 'successful')
run_called_args = { run_called_args = {
'workdir': '/tmp/', 'workdir': '/var/log/validations/artifacts/123_fake.yaml_time',
'playbook': '/tmp/foo/fake.yaml', 'playbook': '/tmp/foo/fake.yaml',
'base_dir': '/usr/share/ansible/', 'base_dir': '/usr/share/ansible/',
'playbook_dir': '/tmp/foo', 'playbook_dir': '/tmp/foo',
@ -135,8 +160,8 @@ class TestValidationActions(TestCase):
'extra_env_variables': None, 'extra_env_variables': None,
'ansible_cfg': None, 'ansible_cfg': None,
'gathering_policy': 'explicit', 'gathering_policy': 'explicit',
'ansible_artifact_path': '/tmp/', 'ansible_artifact_path': '/var/log/validations/artifacts/123_fake.yaml_time',
'log_path': None, 'log_path': '/var/log/validations',
'run_async': False, 'run_async': False,
'python_interpreter': None, 'python_interpreter': None,
'ssh_user': None 'ssh_user': None
@ -152,40 +177,31 @@ class TestValidationActions(TestCase):
run = ValidationActions() run = ValidationActions()
run_return = run.run_validations(playbook, inventory, run_return = run.run_validations(playbook, inventory,
log_path='/var/log/validations',
validations_dir='/tmp/foo', validations_dir='/tmp/foo',
skip_list=skip_list, skip_list=skip_list,
limit_hosts='cloud,cloud1,!cloud2') limit_hosts='cloud,cloud1,!cloud2')
mock_ansible_run.assert_called_with(**run_called_args) mock_ansible_run.assert_called_with(**run_called_args)
@mock.patch('validations_libs.validation_logs.ValidationLogs.get_results') @mock.patch('validations_libs.validation_actions.ValidationLogs.get_results',
side_effect=fakes.FAKE_SUCCESS_RUN)
@mock.patch('validations_libs.utils.parse_all_validations_on_disk') @mock.patch('validations_libs.utils.parse_all_validations_on_disk')
@mock.patch('validations_libs.ansible.Ansible.run') @mock.patch('validations_libs.ansible.Ansible.run')
@mock.patch('validations_libs.utils.create_artifacts_dir', def test_validation_run_success(self, mock_ansible_run,
return_value=('1234', '/tmp/')) mock_validation_dir,
def test_validation_run_success(self, mock_tmp, mock_ansible_run, mock_results):
mock_validation_dir, mock_results):
mock_validation_dir.return_value = [{ mock_validation_dir.return_value = [{
'description': 'My Validation One Description', 'description': 'My Validation One Description',
'groups': ['prep', 'pre-deployment'], 'groups': ['prep', 'pre-deployment'],
'id': 'foo', 'id': 'foo',
'name': 'My Validition One Name', 'name': 'My Validition One Name',
'parameters': {}}] 'parameters': {}}]
mock_ansible_run.return_value = ('foo.yaml', 0, 'successful') mock_ansible_run.return_value = ('foo.yaml', 0, 'successful')
mock_results.return_value = [{'Duration': '0:00:01.761', expected_run_return = fakes.FAKE_SUCCESS_RUN[0]
'Host_Group': 'overcloud',
'Status': 'PASSED',
'Status_by_Host': 'subnode-1,PASSED',
'UUID': 'foo',
'Unreachable_Hosts': '',
'Validations': 'ntp'}]
expected_run_return = [{'Duration': '0:00:01.761',
'Host_Group': 'overcloud',
'Status': 'PASSED',
'Status_by_Host': 'subnode-1,PASSED',
'UUID': 'foo',
'Unreachable_Hosts': '',
'Validations': 'ntp'}]
playbook = ['fake.yaml'] playbook = ['fake.yaml']
inventory = 'tmp/inventory.yaml' inventory = 'tmp/inventory.yaml'
@ -209,17 +225,18 @@ class TestValidationActions(TestCase):
@mock.patch('validations_libs.validation_logs.ValidationLogs.get_results') @mock.patch('validations_libs.validation_logs.ValidationLogs.get_results')
@mock.patch('validations_libs.utils.parse_all_validations_on_disk') @mock.patch('validations_libs.utils.parse_all_validations_on_disk')
@mock.patch('validations_libs.ansible.Ansible.run') @mock.patch('validations_libs.ansible.Ansible.run')
@mock.patch('validations_libs.utils.create_artifacts_dir', def test_validation_run_failed(self, mock_ansible_run,
return_value=('1234', '/tmp/'))
def test_validation_run_failed(self, mock_tmp, mock_ansible_run,
mock_validation_dir, mock_results): mock_validation_dir, mock_results):
mock_validation_dir.return_value = [{ mock_validation_dir.return_value = [{
'description': 'My Validation One Description', 'description': 'My Validation One Description',
'groups': ['prep', 'pre-deployment'], 'groups': ['prep', 'pre-deployment'],
'id': 'foo', 'id': 'foo',
'name': 'My Validition One Name', 'name': 'My Validition One Name',
'parameters': {}}] 'parameters': {}}]
mock_ansible_run.return_value = ('foo.yaml', 0, 'failed') mock_ansible_run.return_value = ('foo.yaml', 0, 'failed')
mock_results.return_value = [{'Duration': '0:00:01.761', mock_results.return_value = [{'Duration': '0:00:01.761',
'Host_Group': 'overcloud', 'Host_Group': 'overcloud',
'Status': 'PASSED', 'Status': 'PASSED',
@ -227,6 +244,7 @@ class TestValidationActions(TestCase):
'UUID': 'foo', 'UUID': 'foo',
'Unreachable_Hosts': '', 'Unreachable_Hosts': '',
'Validations': 'ntp'}] 'Validations': 'ntp'}]
expected_run_return = [{'Duration': '0:00:01.761', expected_run_return = [{'Duration': '0:00:01.761',
'Host_Group': 'overcloud', 'Host_Group': 'overcloud',
'Status': 'PASSED', 'Status': 'PASSED',

42
validations_libs/utils.py
View File

@ -32,25 +32,51 @@ def current_time():
return '%sZ' % datetime.datetime.utcnow().isoformat() return '%sZ' % datetime.datetime.utcnow().isoformat()
def create_artifacts_dir(dir_path=None, prefix=None): def create_log_dir(log_path=constants.VALIDATIONS_LOG_BASEDIR):
"""Create Ansible artifacts directory """Create default validations log dir.
Log the failure if encountering OSError or PermissionError.
:raises: RuntimeError
"""
try:
if os.path.exists(log_path):
if os.access(log_path, os.W_OK):
return log_path
else:
create_log_dir(constants.VALIDATIONS_LOG_BASEDIR)
else:
os.makedirs(log_path)
return log_path
except (OSError, PermissionError):
LOG.error(
(
"Error while creating the log directory. "
"Please check the access rights for: '{}'"
).format(log_path)
)
# Fallback in default path if log_path != from constants path
if log_path != constants.VALIDATIONS_LOG_BASEDIR:
create_log_dir(constants.VALIDATIONS_LOG_BASEDIR)
raise RuntimeError()
:param dir_path: Directory asbolute path
:type dir_path: `string` def create_artifacts_dir(log_path=constants.VALIDATIONS_LOG_BASEDIR,
prefix=None):
"""Create Ansible artifacts directory
:param log_path: Directory asbolute path
:type log_path: `string`
:param prefix: Playbook name :param prefix: Playbook name
:type prefix: `string` :type prefix: `string`
:return: The UUID of the validation and the absolute Path of the log file :return: The UUID of the validation and the absolute Path of the log file
:rtype: `string`, `string` :rtype: `string`, `string`
""" """
dir_path = (dir_path if dir_path else artifact_dir = os.path.join(log_path, 'artifacts')
constants.VALIDATION_ANSIBLE_ARTIFACT_PATH)
validation_uuid = str(uuid.uuid4()) validation_uuid = str(uuid.uuid4())
log_dir = "{}/{}_{}_{}".format(dir_path, validation_uuid, log_dir = "{}/{}_{}_{}".format(artifact_dir, validation_uuid,
(prefix if prefix else ''), current_time()) (prefix if prefix else ''), current_time())
try: try:
os.makedirs(log_dir) os.makedirs(log_dir)
return validation_uuid, log_dir return validation_uuid, log_dir
except OSError: except (OSError, PermissionError):
LOG.exception( LOG.exception(
( (
"Error while creating Ansible artifacts log file." "Error while creating Ansible artifacts log file."

10
validations_libs/validation_actions.py
View File

@ -228,7 +228,8 @@ class ValidationActions(object):
extra_env_vars=None, ansible_cfg=None, quiet=True, extra_env_vars=None, ansible_cfg=None, quiet=True,
workdir=None, limit_hosts=None, run_async=False, workdir=None, limit_hosts=None, run_async=False,
base_dir=constants.DEFAULT_VALIDATIONS_BASEDIR, base_dir=constants.DEFAULT_VALIDATIONS_BASEDIR,
log_path=None, python_interpreter=None, log_path=constants.VALIDATIONS_LOG_BASEDIR,
python_interpreter=None,
skip_list=None, skip_list=None,
callback_whitelist=None, callback_whitelist=None,
output_callback='validation_stdout', output_callback='validation_stdout',
@ -268,6 +269,8 @@ class ValidationActions(object):
``constants.DEFAULT_VALIDATIONS_BASEDIR``) ``constants.DEFAULT_VALIDATIONS_BASEDIR``)
:type base_dir: ``string`` :type base_dir: ``string``
:param log_path: The absolute path of the validations logs directory :param log_path: The absolute path of the validations logs directory
(Defaults to
``constants.VALIDATIONS_LOG_BASEDIR``)
:type log_path: ``string`` :type log_path: ``string``
:param python_interpreter: Path to the Python interpreter to be :param python_interpreter: Path to the Python interpreter to be
used for module execution on remote targets, used for module execution on remote targets,
@ -353,6 +356,7 @@ class ValidationActions(object):
else: else:
raise RuntimeError("No validations found") raise RuntimeError("No validations found")
log_path = v_utils.create_log_dir(log_path)
self.log.debug('Running the validations with Ansible') self.log.debug('Running the validations with Ansible')
results = [] results = []
for playbook in playbooks: for playbook in playbooks:
@ -363,7 +367,7 @@ class ValidationActions(object):
limit_hosts) limit_hosts)
if _play: if _play:
validation_uuid, artifacts_dir = v_utils.create_artifacts_dir( validation_uuid, artifacts_dir = v_utils.create_artifacts_dir(
dir_path=log_path, prefix=os.path.basename(playbook)) log_path=log_path, prefix=os.path.basename(playbook))
run_ansible = v_ansible(validation_uuid) run_ansible = v_ansible(validation_uuid)
_playbook, _rc, _status = run_ansible.run( _playbook, _rc, _status = run_ansible.run(
workdir=artifacts_dir, workdir=artifacts_dir,
@ -398,7 +402,7 @@ class ValidationActions(object):
return results return results
# Return log results # Return log results
uuid = [id['UUID'] for id in results] uuid = [id['UUID'] for id in results]
vlog = ValidationLogs() vlog = ValidationLogs(log_path)
return vlog.get_results(uuid) return vlog.get_results(uuid)
def group_information(self, groups): def group_information(self, groups):