openstack
/
tripleo-repos
Code Issues Proposed changes

Enable ansible-test sanity checks (nv) 

Adds non-voting check jobs for ansible-test sanity and fixing many
problems identified by these. We will sort the other problems in
follow-up(s) and make the job voting. This approach will ease
reviewing and prevent creating conflicts with other pending changes.

Story: https://projects.engineering.redhat.com/browse/TRIPLEOCI-559
Change-Id: I29ce27114e0675fb5d00614df506675bfbe5bf96
This commit is contained in:
Sorin Sbarnea 2021-07-23 09:36:51 +01:00
parent 77dc5794ac
commit 1525e5e1fa
17 changed files with 157 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
meta/runtime.yml
View File

@ -1 +1 @@
requires_ansible: 2.9.0
requires_ansible: ">=2.9.0"

5
plugins/module_utils/tripleo_repos/get_hash/constants.py
View File

@ -13,6 +13,11 @@
# under the License.
#
#
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
"""
These are the keys we expect to find in a well-formed config.yaml

4
plugins/module_utils/tripleo_repos/get_hash/exceptions.py
View File

@ -13,6 +13,10 @@
# under the License.
#
#
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
class Base(Exception):

25
plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py
View File

@ -13,14 +13,18 @@
# under the License.
#
#
from __future__ import (absolute_import, division, print_function)
import logging
import sys
import yaml
import os
import yaml
import requests
import tripleo_repos.get_hash.constants as const
import tripleo_repos.get_hash.exceptions as exc
from .constants import CONFIG_PATH, CONFIG_KEYS
from .exceptions import TripleOHashMissingConfig, TripleOHashInvalidConfig
__metaclass__ = type
class TripleOHashInfo:
@ -111,29 +115,29 @@ class TripleOHashInfo:
config_path = ''
local_config = _resolve_local_config_path()
# prefer const.CONFIG_PATH then local_config
if _check_read_file(const.CONFIG_PATH):
config_path = const.CONFIG_PATH
if _check_read_file(CONFIG_PATH):
config_path = CONFIG_PATH
elif local_config:
config_path = local_config
else:
raise exc.TripleOHashMissingConfig(
raise TripleOHashMissingConfig(
"Configuration file not found at {} or {}".format(
const.CONFIG_PATH, local_config
CONFIG_PATH, local_config
)
)
logging.info("Using config file at {}".format(config_path))
with open(config_path, 'r') as config_yaml:
loaded_config = yaml.safe_load(config_yaml)
for k in const.CONFIG_KEYS:
for k in CONFIG_KEYS:
if k not in loaded_config:
error_str = (
"Malformed config file - missing {}. Expected all"
"of these configuration items: {}"
).format(
k, ", ".join(const.CONFIG_KEYS)
k, ", ".join(CONFIG_KEYS)
)
logging.error(error_str)
raise exc.TripleOHashInvalidConfig(error_str)
raise TripleOHashInvalidConfig(error_str)
# if the passed config contains the key then use that value
if passed_config.get(k):
result_config[k] = passed_config[k]
@ -150,7 +154,6 @@ class TripleOHashInfo:
:param tag: The Delorean server named tag e.g. current-tripleo
:param config: Use an existing config dictionary and don't load it
"""
config = TripleOHashInfo.load_config(config)
self.os_version = os_version

10
plugins/module_utils/tripleo_repos/main.py
View File

@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
from __future__ import (absolute_import, division, print_function)
import argparse
import os
import platform
@ -22,9 +22,8 @@ import re
import subprocess
import sys
import requests
__metaclass__ = type
TITLE_RE = re.compile('\\[(.*)\\]')
NAME_RE = re.compile('name=(.+)')
PRIORITY_RE = re.compile('priority=\\d+')
@ -222,6 +221,11 @@ def _parse_args(distro_id, distro_major_version_id):
def _get_repo(path, args):
# lazy import
if 'requests' not in globals():
import requests
r = requests.get(path)
if r.status_code == 200:
return _inject_mirrors(r.text, args)

4
plugins/module_utils/tripleo_repos/yum_config/constants.py
View File

@ -13,10 +13,12 @@
# under the License.
#
#
from __future__ import (absolute_import, division, print_function)
"""
List of options that can be updated for yum repo files.
"""
__metaclass__ = type
YUM_REPO_SUPPORTED_OPTIONS = [
'name',
'baseurl',

5
plugins/module_utils/tripleo_repos/yum_config/dnf_manager.py
View File

@ -11,10 +11,13 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import (absolute_import, division, print_function)
import logging
__metaclass__ = type
class DnfModuleManager:
"""Class that manages dnf modules."""

5
plugins/module_utils/tripleo_repos/yum_config/exceptions.py
View File

@ -13,6 +13,11 @@
# under the License.
#
#
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
class Base(Exception):
"""Base Exception class."""

44
plugins/module_utils/tripleo_repos/yum_config/yum_config.py
View File

@ -12,14 +12,30 @@
# License for the specific language governing permissions and limitations
# under the License.
#
from __future__ import (absolute_import, division, print_function)
import configparser
import logging
import os
import sys
import tripleo_repos.yum_config.constants as const
import tripleo_repos.yum_config.exceptions as exc
from .constants import (
YUM_GLOBAL_CONFIG_FILE_PATH,
YUM_REPO_DIR,
YUM_REPO_FILE_EXTENSION,
YUM_REPO_SUPPORTED_OPTIONS,
)
from .exceptions import (
TripleOYumConfigFileParseError,
TripleOYumConfigInvalidOption,
TripleOYumConfigInvalidSection,
TripleOYumConfigNotFound,
TripleOYumConfigPermissionDenied,
)
__metaclass__ = type
class TripleOYumConfig:
@ -81,23 +97,23 @@ class TripleOYumConfig:
if not (file_path or dir_path):
msg = ('A configuration file path or a directory path must be '
'provided.')
raise exc.TripleOYumConfigNotFound(error_msg=msg)
raise TripleOYumConfigNotFound(error_msg=msg)
if file_path:
if not os.path.isfile(file_path):
msg = ('The configuration file "{}" was not found in the '
'provided path.').format(file_path)
raise exc.TripleOYumConfigNotFound(error_msg=msg)
raise TripleOYumConfigNotFound(error_msg=msg)
if not os.access(file_path, os.W_OK):
msg = ('The configuration file {} is not '
'writable.'.format(file_path))
raise exc.TripleOYumConfigPermissionDenied(error_msg=msg)
raise TripleOYumConfigPermissionDenied(error_msg=msg)
if dir_path:
if not os.path.isdir(dir_path):
msg = ('The configuration dir "{}" was not found in the '
'provided path.').format(dir_path)
raise exc.TripleOYumConfigNotFound(error_msg=msg)
raise TripleOYumConfigNotFound(error_msg=msg)
def _read_config_file(self, section):
"""Read the configuration file associate with this object.
@ -118,13 +134,13 @@ class TripleOYumConfig:
except configparser.Error:
msg = 'Unable to parse configuration file {}.'.format(
self.config_file_path)
raise exc.TripleOYumConfigFileParseError(error_msg=msg)
raise TripleOYumConfigFileParseError(error_msg=msg)
if section not in config.sections():
msg = ('The provided section "{}" was not found in the '
'configuration file {}.').format(
section, self.config_file_path)
raise exc.TripleOYumConfigInvalidSection(error_msg=msg)
raise TripleOYumConfigInvalidSection(error_msg=msg)
return config, self.config_file_path
@ -168,13 +184,13 @@ class TripleOYumConfig:
if self.valid_options:
if not all(key in self.valid_options for key in set_dict.keys()):
msg = 'One or more provided options are not valid.'
raise exc.TripleOYumConfigInvalidOption(error_msg=msg)
raise TripleOYumConfigInvalidOption(error_msg=msg)
config, config_file_path = self._read_config_file(section)
if not (config and config_file_path):
msg = ('The provided section "{}" was not found within any '
'configuration file.').format(section)
raise exc.TripleOYumConfigNotFound(error_msg=msg)
raise TripleOYumConfigNotFound(error_msg=msg)
# Update configuration file with dict updates
config[section].update(set_dict)
@ -192,13 +208,13 @@ class TripleOYumRepoConfig(TripleOYumConfig):
if file_path:
logging.info(
"Using '{}' as yum repo configuration file.".format(file_path))
conf_dir_path = dir_path or const.YUM_REPO_DIR
conf_dir_path = dir_path or YUM_REPO_DIR
super(TripleOYumRepoConfig, self).__init__(
valid_options=const.YUM_REPO_SUPPORTED_OPTIONS,
valid_options=YUM_REPO_SUPPORTED_OPTIONS,
file_path=file_path,
dir_path=conf_dir_path,
file_extension=const.YUM_REPO_FILE_EXTENSION)
file_extension=YUM_REPO_FILE_EXTENSION)
def update_section(self, section, set_dict, enable=None):
if enable is not None:
@ -211,7 +227,7 @@ class TripleOYumGlobalConfig(TripleOYumConfig):
"""Manages yum global configuration file."""
def __init__(self, file_path=None):
conf_file_path = file_path or const.YUM_GLOBAL_CONFIG_FILE_PATH
conf_file_path = file_path or YUM_GLOBAL_CONFIG_FILE_PATH
logging.info("Using '{}' as yum global configuration "
"file.".format(conf_file_path))
if file_path is None:

39
plugins/modules/get_hash.py
View File

@ -1,31 +1,22 @@
#!/usr/bin/python
# Copyright 2021 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# Copyright 2021 Red Hat, Inc.
# GNU General Public License v3.0+ (see COPYING or
# https://www.gnu.org/licenses/gpl-3.0.txt)
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
from tripleo_repos.get_hash.tripleo_hash_info import TripleOHashInfo
from ansible.module_utils.basic import AnsibleModule
DOCUMENTATION = r'''
---
module: tripleo_get_hash
module: get_hash
short_description: Resolve rdo named tag to commit, full and distro hashes
version_added: "2.9"
version_added: "1.0.0"
description:
description: ""
options:
os_version:
@ -42,7 +33,6 @@ options:
description: The tripleo-ci component you are interested in
required: false
type: str
default: None
tag:
description: The named tag to fetch
required: false
@ -95,6 +85,8 @@ dlrn_url:
sample: 'https://trunk.rdoproject.org/centos8-master/current-tripleo/delorean.repo.md5' # noqa E501
'''
from ansible.module_utils.basic import AnsibleModule # noqa: E402
def run_module():
result = dict(
@ -120,6 +112,13 @@ def run_module():
try:
try:
from ansible_collections.tripleo.repos.plugins.module_utils.\
tripleo_repo.get_hash.tripleo_hash_info import TripleOHashInfo
except ImportError:
from tripleo_repos.get_hash.tripleo_hash_info import \
TripleOHashInfo
os_version = module.params.get('os_version')
release = module.params.get('release')
component = module.params.get('component')

38
plugins/modules/yum_config.py
View File

@ -1,30 +1,18 @@
#!/usr/bin/python
# Copyright 2021 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Copyright 2021 Red Hat, Inc.
# GNU General Public License v3.0+ (see COPYING or
# https://www.gnu.org/licenses/gpl-3.0.txt)
from __future__ import (absolute_import, division, print_function)
from ansible.module_utils.basic import AnsibleModule
import tripleo_repos.yum_config.dnf_manager as dnf_mgr
import tripleo_repos.yum_config.yum_config as cfg
__metaclass__ = type
DOCUMENTATION = r'''
---
module: tripleo_yum_config
module: yum_config
short_description: Update yum configuration files for TripleO deployments.
version_added: "2.9"
version_added: "1.0.0"
description:
- Update specific options for different yum configuration files like
@ -134,6 +122,8 @@ EXAMPLES = r'''
keepcache: "0"
'''
from ansible.module_utils.basic import AnsibleModule # noqa: E402
def run_module():
# define available arguments/parameters a user can pass to the module
@ -198,6 +188,16 @@ def run_module():
# Module execution
try:
try:
import ansible_collections.tripleo.repos.plugins.module_utils.\
tripleo_repos.yum_config.dnf_manager as dnf_mgr
import ansible_collections.tripleo.repos.plugins.module_utils.\
tripleo_repos.yum_config.yum_config as cfg
except ImportError:
import tripleo_repos.yum_config.dnf_manager as dnf_mgr
import tripleo_repos.yum_config.yum_config as cfg
if m_type == 'repo':
config_obj = cfg.TripleOYumRepoConfig(
file_path=m_file_path,

1
tests/sanity/ignore-2.10.txt Symbolic link
View File

@ -0,0 +1 @@
ignore-2.9.txt

1
tests/sanity/ignore-2.11.txt Symbolic link
View File

@ -0,0 +1 @@
ignore-2.9.txt

13
tests/sanity/ignore-2.9.txt Normal file
View File

@ -0,0 +1,13 @@
plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py import
plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py pylint:ansible-bad-function
plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py pylint:ansible-format-automatic-specification
plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py pylint:logging-format-interpolation
plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py replace-urlopen # we did it right
plugins/module_utils/tripleo_repos/main.py import:traceback
plugins/module_utils/tripleo_repos/main.py pylint:ansible-bad-function
plugins/module_utils/tripleo_repos/main.py pylint:ansible-format-automatic-specification
plugins/module_utils/tripleo_repos/yum_config/dnf_manager.py pylint:ansible-format-automatic-specification
plugins/module_utils/tripleo_repos/yum_config/dnf_manager.py pylint:logging-format-interpolation
plugins/module_utils/tripleo_repos/yum_config/yum_config.py pylint:ansible-format-automatic-specification
plugins/module_utils/tripleo_repos/yum_config/yum_config.py pylint:logging-format-interpolation
plugins/modules/yum_config.py pylint:ansible-format-automatic-specification

2
tests/sanity/requirements.txt Normal file
View File

@ -0,0 +1,2 @@
PyYAML
requests

4
tox.ini
View File

@ -1,7 +1,9 @@
[tox]
minversion = 3.1.1
skipsdist = True
envlist = py,pep8,packaging
envlist = py,pep8,packaging,sanity
requires =
tox-ansible>=1.5.3
[testenv]
usedevelop = True

28
zuul.d/layout.yaml
View File

@ -1,10 +1,22 @@
- job:
name: tox-sanity-py36
description: Run ansible-test sanity tests on a collection
parent: openstack-tox-py36
vars:
tox_envlist: sanity
- job:
name: tox-sanity-py39
description: Run ansible-test sanity tests on a collection
parent: openstack-tox-py39
vars:
tox_envlist: sanity
- project:
templates:
- check-requirements
- openstack-cover-jobs
- openstack-python3-wallaby-jobs
check:
jobs: &cjobs
jobs:
- openstack-tox-pep8:
vars:
tox_envlist: pep8,packaging
@ -17,8 +29,20 @@
- openstack-tox-py39
- tripleo-build-containers-ubi-8:
dependencies: *deps_unit_lint_cprovider
- tox-sanity-py36:
voting: false
- tox-sanity-py39:
voting: false
gate:
jobs: *cjobs
jobs:
- openstack-tox-pep8:
vars:
tox_envlist: pep8,packaging
- openstack-tox-py39
- tripleo-buildimage-overcloud-full-centos-8:
dependencies: *deps_unit_lint_cprovider
- tripleo-build-containers-ubi-8:
dependencies: *deps_unit_lint_cprovider
post:
jobs:
- publish-openstack-python-branch-tarball