openstack
/
tripleo-repos
Code Issues Proposed changes

Merge "Add module to generate CentOS Compose repos"

This commit is contained in:
Zuul 2021-09-23 15:10:26 +00:00 committed by Gerrit Code Review
parent ddaf9e7471 9bc5d12529
commit bbb81ff8d4
11 changed files with 653 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
docs/yum_config.md
View File

@ -15,8 +15,8 @@ its repository and invoking in command line:
This subcommand lets you enable or disable a repo and sets its configuration options.
The *tripleo-yum-config* module will search for the provided repo name in all *.repo* files at REPO_DIR_PATH.
Optionally, you can provide a dir path where your repo files live or specify the full path of the repo file.
By default REPO_DIR_PATH is set to */etc/yum.repod.d/*.
Optionally, you can provide a path where your repo files live or specify the full path of the repo file.
By default REPO_DIR_PATH is set to */etc/yum.repos.d/*.
Examples:
```
@ -48,6 +48,19 @@ its repository and invoking in command line:
```
sudo python -m tripleo_yum_config global --set-opts keepcache=1 cachedir="/var/cache/dnf"
```
* **enable-compose-repos**
This subcommand will enable a list os yum repos based on the metadata retrieved from the `compose-url`.
The *tripleo-yum-config* module will create new repo files at REPO_DIR_PATH and enable them.
Optionally, you can provide a path where your repo files live, specify the variants that should be created and which repos need to be disabled afterwards.
By default REPO_DIR_PATH is set to */etc/yum.repos.d/*.
Example:
```
sudo python -m tripleo_yum_config enable-compose-repos --compose-url https://composes.centos.org/latest-CentOS-Stream-8/compose/ --release centos-stream-8 --disable-all-conflicting
```
#### Install using setup.py
Installation using python setup.py requires sudo, because the python source

33
molecule/default/converge.yml
View File

@ -2,8 +2,8 @@
- name: Converge
hosts: all
tasks:
- name: "Test yum_config repo"
- name: "Test yum_config repo config"
become: true
tripleo.repos.yum_config:
type: repo
name: appstream
@ -14,12 +14,41 @@
- molecule-idempotence-notest
when: ansible_distribution_major_version is version(8, '>=')
- name: "Test yum_config global config"
become: true
tripleo.repos.yum_config:
type: global
file_path: /etc/dnf/dnf.conf
set_options:
skip_if_unavailable: "False"
keepcache: "0"
tags:
# TODO: fix yum_config to correctly report changed state and uncomment
# the line below which disables molecule idemptotence test.
- molecule-idempotence-notest
- name: "Check get_hash"
tripleo.repos.get_hash:
release: master
- name: "Check get_hash with invalid url"
tripleo.repos.get_hash:
release: master
dlrn_url: 'https://httpbin.org/status/404'
register: result
failed_when: result is success
- name: "Test yum_config enable-compose-repos"
become: true
tripleo.repos.yum_config:
type: enable-compose-repos
compose_url: https://composes.centos.org/latest-CentOS-Stream-8/compose/
centos_release: centos-stream-8
variants:
- AppStream
- BaseOS
disable_repos:
- /etc/yum.repos.d/CentOS-Stream-AppStream.repo
- /etc/yum.repos.d/CentOS-Stream-BaseOS.repo
tags:
- molecule-idempotence-notest

80
plugins/module_utils/tripleo_repos/yum_config/__main__.py
View File

@ -17,8 +17,10 @@ import argparse
import logging
import sys
import tripleo_repos.yum_config.yum_config as cfg
import tripleo_repos.yum_config.compose_repos as compose_repos
import tripleo_repos.yum_config.constants as const
import tripleo_repos.yum_config.dnf_manager as dnf_mgr
import tripleo_repos.yum_config.yum_config as cfg
def options_to_dict(options):
@ -63,8 +65,9 @@ def main():
repo_args_parser.add_argument(
'--config-dir-path',
dest='config_dir_path',
default=const.YUM_REPO_DIR,
help=(
'set the absolute directory path that holds all repo or module '
'set the absolute directory path that holds all repo '
'configuration files')
)
@ -98,13 +101,61 @@ def main():
help="sets module profile"
)
# Compose repo arguments
compose_args_parser = argparse.ArgumentParser(add_help=False)
compose_args_parser.add_argument(
'--compose-url',
dest='compose_url',
required=True,
help='CentOS compose URL'
)
compose_args_parser.add_argument(
'--release',
dest='release',
choices=const.COMPOSE_REPOS_RELEASES,
default='centos-stream-8',
help='target CentOS release.'
)
compose_args_parser.add_argument(
'--arch',
choices=const.COMPOSE_REPOS_SUPPORTED_ARCHS,
default='x86_64',
help='set the architecture for the destination repos.'
)
compose_args_parser.add_argument(
'--disable-repos',
nargs='+',
help='list of repo names or repo absolute file paths to be disabled.'
)
compose_args_parser.add_argument(
'--disable-all-conflicting',
action='store_true',
dest='disable_conflicting',
default=False,
help='after enabling compose repos, disable all other repos that '
'match variant names.'
)
compose_args_parser.add_argument(
'--variants',
nargs='+',
help='Name of the repos to be enabled. Default behavior is to enable '
'all that match a specific release and architecture.'
)
compose_args_parser.add_argument(
'--config-dir-path',
dest='config_dir_path',
default=const.YUM_REPO_DIR,
help='set the absolute directory path that holds all repo '
'configuration files'
)
# Common file path argument
common_parse = argparse.ArgumentParser(add_help=False)
common_parse.add_argument(
'--config-file-path',
dest='config_file_path',
help=('set the absolute file path of the configuration file to be '
'updated')
'updated.')
)
# Main parser
@ -133,6 +184,11 @@ def main():
parents=[common_parse, options_parse],
help='updates global yum configuration options'
)
subparsers.add_parser(
'enable-compose-repos',
parents=[compose_args_parser],
help='enable CentOS compose repos based on an compose url.'
)
args = main_parser.parse_args()
if args.command is None:
@ -146,10 +202,11 @@ def main():
if args.command == 'repo':
set_dict = options_to_dict(args.set_opts)
config_obj = cfg.TripleOYumRepoConfig(
file_path=args.config_file_path,
dir_path=args.config_dir_path)
config_obj.update_section(args.name, set_dict, enable=args.enable)
config_obj.update_section(args.name, set_dict,
file_path=args.config_file_path,
enabled=args.enable)
elif args.command == 'module':
dnf_mod_mgr = dnf_mgr.DnfModuleManager()
@ -163,6 +220,19 @@ def main():
config_obj.update_section('main', set_dict)
elif args.command == 'enable-compose-repos':
repo_obj = compose_repos.TripleOYumComposeRepoConfig(
args.compose_url,
args.release,
dir_path=args.config_dir_path,
arch=args.arch)
repo_obj.enable_compose_repos(variants=args.variants,
override_repos=args.disable_conflicting)
if args.disable_repos:
for file in args.disable_repos:
repo_obj.update_all_sections(file, enabled=False)
def cli_entrypoint():
try:

202
plugins/module_utils/tripleo_repos/yum_config/compose_repos.py Normal file
View File

@ -0,0 +1,202 @@
# Copyright 2021 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from __future__ import (absolute_import, division, print_function)
import logging
import json
import os
import re
import urllib.parse
import urllib.request
from .constants import (
YUM_REPO_DIR,
YUM_REPO_FILE_EXTENSION,
YUM_REPO_SUPPORTED_OPTIONS,
COMPOSE_REPOS_RELEASES,
COMPOSE_REPOS_INFO_PATH,
COMPOSE_REPOS_URL_PATTERN,
COMPOSE_REPOS_URL_REPLACE_STR,
)
from .exceptions import (
TripleOYumConfigInvalidSection,
TripleOYumConfigComposeError,
)
from .yum_config import (
TripleOYumConfig
)
__metaclass__ = type
class TripleOYumComposeRepoConfig(TripleOYumConfig):
"""Manages yum repo configuration files for CentOS Compose."""
def __init__(self, compose_url, release, dir_path=None, arch=None):
conf_dir_path = dir_path or YUM_REPO_DIR
self.arch = arch or 'x86_64'
# 1. validate release name
if release not in COMPOSE_REPOS_RELEASES:
msg = 'CentOS release not supported.'
raise TripleOYumConfigComposeError(error_msg=msg)
self.release = release
# 2. Validate URL
pattern = re.compile(COMPOSE_REPOS_URL_PATTERN[self.release])
if not pattern.match(compose_url):
msg = 'The provided URL does not match the expect pattern.'
raise TripleOYumConfigComposeError(error_msg=msg)
# 3. Get compose info from url
segments = [compose_url,
COMPOSE_REPOS_INFO_PATH[self.release]]
self.compose_info_url = '/'.join(s.strip('/') for s in segments)
self.compose_info = self._get_compose_info()
# 4. Get compose-id from metadata
self.compose_id = self.compose_info['compose']['id']
# 5. Replace the compose-id from url to avoid 'labels'
repl_args = {'compose_id': self.compose_id}
self.compose_url = (
pattern.sub(
COMPOSE_REPOS_URL_REPLACE_STR[self.release] % repl_args,
compose_url)
)
super(TripleOYumComposeRepoConfig, self).__init__(
valid_options=YUM_REPO_SUPPORTED_OPTIONS,
dir_path=conf_dir_path,
file_extension=YUM_REPO_FILE_EXTENSION)
def _get_compose_info(self):
"""Retrieve compose info for a provided compose-id url."""
# NOTE(dviroel): works for both centos 8 and 9
try:
logging.debug("Retrieving compose info from url: %s",
self.compose_info_url)
res = urllib.request.urlopen(self.compose_info_url)
except Exception:
msg = ("Failed to retrieve compose info from url: %s"
% self.compose_info_url)
raise TripleOYumConfigComposeError(error_msg=msg)
compose_info = json.loads(res.read())
if compose_info['header']['version'] != "1.2":
# NOTE(dviroel): Log a warning just in case we receive a different
# version here. Code may fail depending on the change.
logging.warning("Expecting compose info version '1.2' but got %s.",
compose_info['header']['version'])
return compose_info['payload']
def _get_repo_name(self, variant):
return " ".join([self.compose_id, variant])
def _get_repo_filename(self, variant):
return "-".join([self.compose_id, variant]) + '.repo'
def _get_repo_base_url(self, variant):
"""Build the base_url based on variant name and system architecture."""
variant_info = self.compose_info['variants'][variant]
if not variant_info['paths'].get('repository', {}).get(self.arch):
# Variant has no support yet
return None
segments = [self.compose_url,
variant_info['paths']['repository'][self.arch]]
return '/'.join(s.strip('/') for s in segments)
def get_compose_variants(self):
return self.compose_info['variants'].keys()
def enable_compose_repos(self, variants=None, override_repos=False):
"""Enable CentOS compose repos of a given variant list.
This function will build from scratch all repos for a given compose-id
url. If a list of variants is not provided, it will enable all for all
variants returned from compose info.
:param variants: A list of variant names to be enabled.
:param override_repos: True if all matching variants in the same
repo directory should be disable in favor of the new repos.
"""
if variants:
for var in variants:
if not (var in self.compose_info['variants'].keys()):
msg = 'One or more provided variants are invalid.'
raise TripleOYumConfigComposeError(error_msg=msg)
else:
variants = self.compose_info['variants'].keys()
updated_repos = {}
for var in variants:
base_url = self._get_repo_base_url(var)
if not base_url:
continue
add_dict = {
'name': self._get_repo_name(var),
'baseurl': base_url,
'enabled': '1',
'gpgcheck': '0',
}
filename = self._get_repo_filename(var)
file_path = os.path.join(self.dir_path, filename)
# create a file if doesn't exist and add a section to it
try:
self.add_section(var.lower(), add_dict, file_path)
except TripleOYumConfigInvalidSection:
logging.debug("Section '%s' that already exists in this file. "
"Skipping...", var)
# needed to override other repos
updated_repos[var.lower()] = file_path
if override_repos:
for var in updated_repos:
config_files = self._get_config_files(var)
for file in config_files:
if file != updated_repos[var]:
msg = ("Disabling matching section '%(section)s' in "
"configuration file: %(file)s.")
msg_args = {
'section': var,
'file': file,
}
logging.debug(msg, msg_args)
self.update_section(var, enabled=False, file_path=file)
def add_section(self, section, add_dict, file_path):
# Create a new file if it does not exists
if not os.path.isfile(file_path):
with open(file_path, '+w'):
pass
super(TripleOYumComposeRepoConfig, self).add_section(
section, add_dict, file_path)
def update_section(
self, section, set_dict=None, enabled=None, file_path=None):
update_dict = set_dict or {}
if enabled is not None:
update_dict['enabled'] = '1' if enabled else '0'
if update_dict:
super(TripleOYumComposeRepoConfig, self).update_section(
section, update_dict, file_path=file_path)
def update_all_sections(self, file_path, set_dict=None, enabled=None):
update_dict = set_dict or {}
if enabled is not None:
update_dict['enabled'] = '1' if enabled else '0'
if update_dict:
super(TripleOYumComposeRepoConfig, self).update_all_sections(
update_dict, file_path)

29
plugins/module_utils/tripleo_repos/yum_config/constants.py
View File

@ -39,3 +39,32 @@ YUM_REPO_FILE_EXTENSION = '.repo'
Default constants for yum/dnf global configurations.
"""
YUM_GLOBAL_CONFIG_FILE_PATH = '/etc/yum.conf'
"""
CentOS Stream compose repos defaults
"""
COMPOSE_REPOS_RELEASES = [
"centos-stream-8",
"centos-stream-9"
]
COMPOSE_REPOS_SUPPORTED_ARCHS = [
"aarch64",
"ppc64le",
"x86_64"
]
COMPOSE_REPOS_URL_PATTERN = {
"centos-stream-8": r"(^https:.*.centos.org/)([^/]*)(/compose/?$)",
"centos-stream-9": r"(^https:.*.centos.org/.*/)(.*)(/compose/?$)",
}
COMPOSE_REPOS_URL_REPLACE_STR = {
"centos-stream-8": r"\1%(compose_id)s\3",
"centos-stream-9": r"\1%(compose_id)s\3",
}
COMPOSE_REPOS_INFO_PATH = {
"centos-stream-8": "metadata/composeinfo.json",
"centos-stream-9": "metadata/composeinfo.json",
}

7
plugins/module_utils/tripleo_repos/yum_config/exceptions.py
View File

@ -60,3 +60,10 @@ class TripleOYumConfigInvalidOption(Base):
def __init__(self, error_msg):
super(TripleOYumConfigInvalidOption, self).__init__(error_msg)
class TripleOYumConfigComposeError(Base):
"""An error occurred while configuring CentOS compose repos."""
def __init__(self, error_msg):
super(TripleOYumConfigComposeError, self).__init__(error_msg)

210
plugins/module_utils/tripleo_repos/yum_config/yum_config.py
View File

@ -31,13 +31,18 @@ from .exceptions import (
TripleOYumConfigInvalidOption,
TripleOYumConfigInvalidSection,
TripleOYumConfigNotFound,
TripleOYumConfigPermissionDenied,
)
__metaclass__ = type
def validated_file_path(file_path):
if os.path.isfile(file_path) and os.access(file_path, os.W_OK):
return True
return False
class TripleOYumConfig:
"""
This class is a base class for updating yum configuration files in
@ -74,79 +79,79 @@ class TripleOYumConfig:
logger.addHandler(handler)
logger.setLevel(logging.INFO)
def __init__(self, valid_options=None, file_path=None, dir_path=None,
file_extension=None):
def __init__(self, valid_options=None, dir_path=None, file_extension=None):
"""
Creates a TripleOYumConfig object that holds configuration file
information.
:param valid_options: A list of options that can be updated on this
file.
:param file_path: The file path to configuration file to be updated.
:param dir_path: The directory path that this class can use to search
for configuration files to be updated.
:param: file_extension: File extension to filter configuration files
in the search directory.
"""
self.config_file_path = file_path
self.dir_path = dir_path
self.file_extension = file_extension
self.valid_options = valid_options
# Sanity checks
if not (file_path or dir_path):
msg = ('A configuration file path or a directory path must be '
'provided.')
raise TripleOYumConfigNotFound(error_msg=msg)
if file_path:
if not os.path.isfile(file_path):
msg = ('The configuration file "{0}" was not found in the '
'provided path.').format(file_path)
raise TripleOYumConfigNotFound(error_msg=msg)
if not os.access(file_path, os.W_OK):
msg = ('The configuration file {0} is not '
'writable.'.format(file_path))
raise TripleOYumConfigPermissionDenied(error_msg=msg)
if dir_path:
if not os.path.isdir(dir_path):
msg = ('The configuration dir "{0}" was not found in the '
'provided path.').format(dir_path)
raise TripleOYumConfigNotFound(error_msg=msg)
def _read_config_file(self, section):
"""Read the configuration file associate with this object.
def _read_config_file(self, file_path, section=None):
"""Reads a configuration file.
If no configuration file is provided, this method will search for
'section' name in all files inside the configuration directory. The
first occurrence of 'section' will be returned, and a warning will be
logged if more than one configuration file has the same 'section' set.
:param section: The name of the section to be updated.
:return: a config parser object and the file path.
:param section: The name of the section that will be update. Only used
to fail earlier if the section is not found.
:return: a config parser object and the full file path.
"""
config = configparser.ConfigParser()
# A) A configuration file path was provided.
if self.config_file_path:
file_paths = [file_path]
if self.dir_path:
# if dir_path is configured, we can search for filename there
file_paths.append(os.path.join(self.dir_path, file_path))
valid_file_path = None
for file in file_paths:
if validated_file_path(file):
valid_file_path = file
break
if not valid_file_path:
msg = ('The configuration file "{0}" was '
'not found.'.format(file_path))
raise TripleOYumConfigNotFound(error_msg=msg)
try:
config.read(self.config_file_path)
config.read(valid_file_path)
except configparser.Error:
msg = 'Unable to parse configuration file {0}.'.format(
self.config_file_path)
valid_file_path)
raise TripleOYumConfigFileParseError(error_msg=msg)
if section not in config.sections():
if section and section not in config.sections():
msg = ('The provided section "{0}" was not found in the '
'configuration file {1}.').format(
section, self.config_file_path)
section, valid_file_path)
raise TripleOYumConfigInvalidSection(error_msg=msg)
return config, self.config_file_path
return config, valid_file_path
# B) Search for a configuration file that has the provided section
section_found = False
config_file_path = None
def _get_config_files(self, section):
"""Gets all configuration file paths for a given section.
This method will search for a 'section' name in all files inside the
configuration directory. All files with 'section' will be returned.
:param section: Section to be found inside configuration files.
:return: A list of config file paths.
"""
# Search for a configuration file that has the provided section
config_files_path = []
if section and self.dir_path:
for file in os.listdir(self.dir_path):
# Skip files that don't match the file extension or are not
# writable
@ -162,83 +167,142 @@ class TripleOYumConfig:
except configparser.Error:
continue
if section in tmp_config.sections():
if section_found:
logging.warning('Section "%s" is listed more than once in '
'configuration files.', section)
else:
# Read the first occurrence of 'section'
config_file_path = os.path.join(self.dir_path, file)
config.read(config_file_path)
section_found = True
config_files_path.append(os.path.join(self.dir_path, file))
return config, config_file_path
return config_files_path
def update_section(self, section, set_dict):
"""Updates a set of options for a specified section.
def update_section(self, section, set_dict, file_path=None):
"""Updates a set of options of a section.
If a file path is not provided by the caller, this function will search
for the section in all files located in the working directory and
update each one of them.
:param section: Name of the section on the configuration file that will
be updated.
:param set_dict: Dict with all options and values to be updated in the
configuration file section.
:param file_path: Path to the configuration file to be updated.
"""
if self.valid_options:
if not all(key in self.valid_options for key in set_dict.keys()):
msg = 'One or more provided options are not valid.'
raise TripleOYumConfigInvalidOption(error_msg=msg)
config, config_file_path = self._read_config_file(section)
if not (config and config_file_path):
msg = ('The provided section "{0}" was not found within any '
'configuration file.').format(section)
files = [file_path] if file_path else self._get_config_files(section)
if not files:
msg = ('No configuration files were found for the provided '
'section {0}'.format(section))
raise TripleOYumConfigNotFound(error_msg=msg)
for file in files:
config, file = self._read_config_file(file, section=section)
# Update configuration file with dict updates
config[section].update(set_dict)
with open(file, 'w') as f:
config.write(f)
with open(config_file_path, 'w') as file:
logging.info("Section '%s' was successfully "
"updated.", section)
def add_section(self, section, add_dict, file_path):
""" Adds a new section with options in a provided config file.
:param section: Section name to be added to the config file.
:param add_dict: Dict with all options and values to be added into the
new section.
:param file_path: Path to the configuration file to be updated.
"""
# This section shouldn't exist in the provided file
config, file_path = self._read_config_file(file_path=file_path)
if section in config.sections():
msg = ("Section '%s' already exists in the configuration "
"file.", section)
raise TripleOYumConfigInvalidSection(error_msg=msg)
# Add new section
config.add_section(section)
# Update configuration file with dict updates
config[section].update(add_dict)
with open(file_path, '+w') as file:
config.write(file)
logging.info("Section '%s' was successfully updated.", section)
logging.info("Section '%s' was successfully "
"added.", section)
def update_all_sections(self, set_dict, file_path):
"""Updates all section of a given configuration file.
:param set_dict: Dict with all options and values to be updated in
the configuration file.
:param file_path: Path to the configuration file to be updated.
"""
if self.valid_options:
if not all(key in self.valid_options for key in set_dict.keys()):
msg = 'One or more provided options are not valid.'
raise TripleOYumConfigInvalidOption(error_msg=msg)
config, file_path = self._read_config_file(file_path)
for section in config.sections():
config[section].update(set_dict)
with open(file_path, '+w') as file:
config.write(file)
logging.info("All sections for '%s' were successfully "
"updated.", file_path)
class TripleOYumRepoConfig(TripleOYumConfig):
"""Manages yum repo configuration files."""
def __init__(self, file_path=None, dir_path=None):
if file_path:
logging.info(
"Using '%s' as yum repo configuration file.", file_path)
def __init__(self, dir_path=None):
conf_dir_path = dir_path or YUM_REPO_DIR
super(TripleOYumRepoConfig, self).__init__(
valid_options=YUM_REPO_SUPPORTED_OPTIONS,
file_path=file_path,
dir_path=conf_dir_path,
file_extension=YUM_REPO_FILE_EXTENSION)
def update_section(self, section, set_dict, enable=None):
if enable is not None:
set_dict['enabled'] = '1' if enable else '0'
super(TripleOYumRepoConfig, self).update_section(section, set_dict)
def update_section(
self, section, set_dict=None, file_path=None, enabled=None):
update_dict = set_dict or {}
if enabled is not None:
update_dict['enabled'] = '1' if enabled else '0'
if update_dict:
super(TripleOYumRepoConfig, self).update_section(
section, update_dict, file_path=file_path)
class TripleOYumGlobalConfig(TripleOYumConfig):
"""Manages yum global configuration file."""
def __init__(self, file_path=None):
conf_file_path = file_path or YUM_GLOBAL_CONFIG_FILE_PATH
self.conf_file_path = file_path or YUM_GLOBAL_CONFIG_FILE_PATH
logging.info("Using '%s' as yum global configuration "
"file.", conf_file_path)
if file_path is None:
"file.", self.conf_file_path)
if file_path is not None:
# validate user provided file path
validated_file_path(file_path)
else:
# If there is no default 'yum.conf' configuration file, we need to
# create it. If the user specify another conf file that doesn't
# exists, the operation will fail.
if not os.path.isfile(conf_file_path):
if not os.path.isfile(self.conf_file_path):
config = configparser.ConfigParser()
config.read(conf_file_path)
config.read(self.conf_file_path)
config.add_section('main')
with open(conf_file_path, '+w') as file:
with open(self.conf_file_path, '+w') as file:
config.write(file)
super(TripleOYumGlobalConfig, self).__init__(file_path=conf_file_path)
super(TripleOYumGlobalConfig, self).__init__()
def update_section(self, section, set_dict, file_path=None):
super(TripleOYumGlobalConfig, self).update_section(
section, set_dict, file_path=(file_path or self.conf_file_path))
def add_section(self, section, set_dict, file_path=None):
add_file_path = file_path or self.conf_file_path
super(TripleOYumGlobalConfig, self).add_section(
section, set_dict, add_file_path)

93
plugins/modules/yum_config.py
View File

@ -24,7 +24,7 @@ options:
- The type of yum configuration to be changed.
required: true
type: str
choices: [repo, module, global]
choices: [repo, module, global, 'enable-compose-repos']
name:
description:
- Name of the repo or module to be changed. This options is
@ -65,6 +65,40 @@ options:
- Absolute path of the directory that contains the configuration
file to be changed.
type: path
default: /etc/yum.repos.d
compose_url:
description:
- URL that contains CentOS compose repositories.
type: str
centos_release:
description:
- Target CentOS release.
type: str
choices: [centos-stream-8, centos-stream-9]
arch:
description:
- System architecture which the repos will be configure.
type: str
choices: [aarch64, ppc64le, x86_64]
default: x86_64
variants:
description:
- Repository variants that should be configured. If not provided,
all available variants will be configured.
type: list
elements: str
disable_conflicting_variants:
description:
- Disable all repos from the same directory that match variants'
name.
type: bool
default: false
disable_repos:
description:
- List with file path of repos that should be disabled after
successfully enabling all compose repos.
type: list
elements: str
author:
- Douglas Viroel (@viroel)
@ -113,6 +147,20 @@ EXAMPLES = r'''
set_options:
skip_if_unavailable: "False"
keepcache: "0"
- name: Configure a set of repos based on latest CentOS Stream 8 compose
become: true
become_user: root
tripleo_yup_config:
compose_url: https://composes.centos.org/latest-CentOS-Stream-8/compose/
centos_release: centos-stream-8
variants:
- AppStream
- BaseOS
disable_conflicting_variants: true
disable_repos:
- /etc/yum.repos.d/CentOS-Linux-AppStream.repo
- /etc/yum.repos.d/CentOS-Linux-BaseOS.repo
'''
RETURN = r''' # '''
@ -121,8 +169,14 @@ from ansible.module_utils.basic import AnsibleModule # noqa: E402
def run_module():
try:
import ansible_collections.tripleo.repos.plugins.module_utils. \
tripleo_repos.yum_config.constants as const
except ImportError:
import tripleo_repos.yum_config.constants as const
# define available arguments/parameters a user can pass to the module
supported_config_types = ['repo', 'module', 'global']
supported_config_types = ['repo', 'module', 'global',
'enable-compose-repos']
supported_module_operations = ['install', 'remove', 'reset']
module_args = dict(
type=dict(type='str', required=True, choices=supported_config_types),
@ -133,7 +187,17 @@ def run_module():
profile=dict(type='str'),
set_options=dict(type='dict', default={}),
file_path=dict(type='path'),
dir_path=dict(type='path'),
dir_path=dict(type='path', default=const.YUM_REPO_DIR),
compose_url=dict(type='str'),
centos_release=dict(type='str',
choices=const.COMPOSE_REPOS_RELEASES),
arch=dict(type='str', choices=const.COMPOSE_REPOS_SUPPORTED_ARCHS,
default='x86_64'),
variants=dict(type='list', default=[],
elements='str'),
disable_conflicting_variants=dict(type='bool', default=False),
disable_repos=dict(type='list', default=[],
elements='str'),
)
module = AnsibleModule(
@ -141,6 +205,7 @@ def run_module():
required_if=[
["type", "repo", ["name"]],
["type", "module", ["name"]],
["type", "enable-compose-repos", ["compose_url"]],
],
supports_check_mode=False
)
@ -162,18 +227,36 @@ def run_module():
tripleo_repos.yum_config.dnf_manager as dnf_mgr
import ansible_collections.tripleo.repos.plugins.module_utils.\
tripleo_repos.yum_config.yum_config as cfg
import ansible_collections.tripleo.repos.plugins.module_utils. \
tripleo_repos.yum_config.compose_repos as repos
except ImportError:
import tripleo_repos.yum_config.dnf_manager as dnf_mgr
import tripleo_repos.yum_config.yum_config as cfg
import tripleo_repos.yum_config.compose_repos as repos
if module.params['type'] == 'repo':
config_obj = cfg.TripleOYumRepoConfig(
file_path=module.params['file_path'],
dir_path=module.params['dir_path'])
config_obj.update_section(
module.params['name'],
m_set_opts,
enable=module.params['enabled'])
file_path=module.params['file_path'],
enabled=module.params['enabled'])
elif module.params['type'] == 'enable-compose-repos':
# 1. Create compose repo config object
repo_obj = repos.TripleOYumComposeRepoConfig(
module.params['compose_url'],
module.params['centos_release'],
dir_path=module.params['dir_path'],
arch=module.params['arch'])
# 2. enable CentOS compose repos
repo_obj.enable_compose_repos(
variants=module.params['variants'],
override_repos=module.params['disable_conflicting_variants'])
# 3. Disable all repos provided in disable_repos
for file in module.params['disable_repos']:
repo_obj.update_all_sections(file, enabled=False)
elif module.params['type'] == 'module':
dnf_mod_mgr = dnf_mgr.DnfModuleManager()

1
tests/sanity/ignore.txt
View File

@ -1,2 +1,3 @@
plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py replace-urlopen
plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py pylint:ansible-bad-import
plugins/module_utils/tripleo_repos/yum_config/compose_repos.py replace-urlopen

9
tests/unit/yum_config/test_main.py
View File

@ -20,8 +20,9 @@ from unittest import mock
from . import fakes
from . import mock_modules # noqa: F401
import tripleo_repos.yum_config.__main__ as main
import tripleo_repos.yum_config.yum_config as yum_cfg
import tripleo_repos.yum_config.constants as const
import tripleo_repos.yum_config.dnf_manager as dnf_mgr
import tripleo_repos.yum_config.yum_config as yum_cfg
class TestTripleoYumConfigBase(unittest.TestCase):
@ -56,10 +57,10 @@ class TestTripleoYumConfigMain(TestTripleoYumConfigBase):
main.main()
expected_dict = {'key1': 'value1', 'key2': 'value2'}
mock_yum_repo_obj.assert_called_once_with(
file_path=fakes.FAKE_FILE_PATH, dir_path=None)
mock_yum_repo_obj.assert_called_once_with(dir_path=const.YUM_REPO_DIR)
mock_update_section.assert_called_once_with(
'fake_repo', expected_dict, enable=True)
'fake_repo', expected_dict, file_path=fakes.FAKE_FILE_PATH,
enabled=True)
@ddt.data('enable', 'disable', 'reset', 'install', 'remove')
def test_main_module(self, operation):

126
tests/unit/yum_config/test_yum_config.py
View File

@ -29,51 +29,25 @@ import tripleo_repos.yum_config.yum_config as yum_cfg
class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase):
"""Tests for TripleYumConfig class and its methods."""
def _create_yum_config_obj(self, file_path=None, dir_path=None,
valid_options=None, file_extension=None):
def _create_yum_config_obj(self, dir_path=None, valid_options=None,
file_extension=None):
self.mock_object(os.path, 'isfile')
self.mock_object(os, 'access')
self.mock_object(os.path, 'isdir')
return yum_cfg.TripleOYumConfig(file_path=file_path, dir_path=dir_path,
return yum_cfg.TripleOYumConfig(dir_path=dir_path,
valid_options=valid_options,
file_extension=file_extension)
@ddt.data(
{'file_path': None, 'dir_path': None, 'is_file_ret': None,
'access_ret': None, 'is_dir_ret': None,
'exception': exc.TripleOYumConfigNotFound},
{'file_path': 'fake_path', 'dir_path': None, 'is_file_ret': False,
'access_ret': None, 'is_dir_ret': None,
'exception': exc.TripleOYumConfigNotFound},
{'file_path': 'fake_path', 'dir_path': None, 'is_file_ret': True,
'access_ret': False, 'is_dir_ret': None,
'exception': exc.TripleOYumConfigPermissionDenied},
{'file_path': None, 'dir_path': 'fake_dir', 'is_file_ret': None,
'access_ret': None, 'is_dir_ret': False,
'exception': exc.TripleOYumConfigNotFound},
)
@ddt.unpack
def test_tripleo_yum_config_invalid_parameters(
self, file_path, dir_path, is_file_ret, access_ret, is_dir_ret,
exception):
self.mock_object(os.path, 'isfile',
mock.Mock(return_value=is_file_ret))
self.mock_object(os, 'access',
mock.Mock(return_value=access_ret))
def test_tripleo_yum_config_invalid_dir_path(self):
self.mock_object(os.path, 'isdir',
mock.Mock(return_value=is_dir_ret))
mock.Mock(return_value=False))
self.assertRaises(exception,
self.assertRaises(exc.TripleOYumConfigNotFound,
yum_cfg.TripleOYumConfig,
file_path=file_path,
dir_path=dir_path)
dir_path='fake_dir_path')
def test_read_config_file_path(self):
yum_config = self._create_yum_config_obj(
file_path=fakes.FAKE_FILE_PATH)
yum_config = self._create_yum_config_obj()
parser_mock = mock.Mock()
self.mock_object(configparser, 'ConfigParser',
@ -83,7 +57,8 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase):
mock.Mock(return_value=fakes.FAKE_SECTIONS))
config_parser, file_path = yum_config._read_config_file(
fakes.FAKE_SECTION1
fakes.FAKE_FILE_PATH,
section=fakes.FAKE_SECTION1
)
self.assertEqual(parser_mock, config_parser)
@ -91,8 +66,7 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase):
read_mock.assert_called_once_with(fakes.FAKE_FILE_PATH)
def test_read_config_file_path_parse_error(self):
yum_config = self._create_yum_config_obj(
file_path=fakes.FAKE_FILE_PATH)
yum_config = self._create_yum_config_obj()
parser_mock = mock.Mock()
self.mock_object(configparser, 'ConfigParser',
@ -102,13 +76,13 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase):
self.assertRaises(exc.TripleOYumConfigFileParseError,
yum_config._read_config_file,
fakes.FAKE_SECTION1)
fakes.FAKE_FILE_PATH,
section=fakes.FAKE_SECTION1)
read_mock.assert_called_once_with(fakes.FAKE_FILE_PATH)
def test_read_config_file_path_invalid_section(self):
yum_config = self._create_yum_config_obj(
file_path=fakes.FAKE_FILE_PATH)
yum_config = self._create_yum_config_obj()
parser_mock = mock.Mock()
self.mock_object(configparser, 'ConfigParser',
@ -119,11 +93,12 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase):
self.assertRaises(exc.TripleOYumConfigInvalidSection,
yum_config._read_config_file,
'invalid_section')
fakes.FAKE_FILE_PATH,
section='invalid_section')
read_mock.assert_called_once_with(fakes.FAKE_FILE_PATH)
def test_read_config_file_dir(self):
def test_get_config_files(self):
yum_config = self._create_yum_config_obj(
dir_path=fakes.FAKE_DIR_PATH,
file_extension='.conf')
@ -133,50 +108,28 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase):
parser_mocks.append(parser_mock)
self.mock_object(parser_mock, 'read')
self.mock_object(parser_mocks[1], 'sections',
self.mock_object(parser_mocks[0], 'sections',
mock.Mock(return_value=[]))
# second file inside dir will have the expected sections
self.mock_object(parser_mocks[2], 'sections',
self.mock_object(parser_mocks[1], 'sections',
mock.Mock(return_value=fakes.FAKE_SECTIONS))
self.mock_object(parser_mocks[2], 'sections',
mock.Mock(return_value=[]))
self.mock_object(os, 'listdir',
mock.Mock(return_value=fakes.FAKE_DIR_FILES))
self.mock_object(os, 'access', mock.Mock(return_value=True))
self.mock_object(configparser, 'ConfigParser',
mock.Mock(side_effect=parser_mocks))
config_parser, file_path = yum_config._read_config_file(
fakes.FAKE_SECTION1)
expected_dir_path = os.path.join(fakes.FAKE_DIR_PATH,
fakes.FAKE_DIR_FILES[1])
result = yum_config._get_config_files(fakes.FAKE_SECTION1)
expected_dir_path = [os.path.join(fakes.FAKE_DIR_PATH,
fakes.FAKE_DIR_FILES[1])]
self.assertEqual(parser_mocks[0], config_parser)
self.assertEqual(expected_dir_path, file_path)
def test_read_config_file_dir_section_not_found(self):
yum_config = self._create_yum_config_obj(
dir_path=fakes.FAKE_DIR_PATH,
file_extension='.conf')
parser_mock = mock.Mock()
self.mock_object(parser_mock, 'read')
self.mock_object(parser_mock, 'sections',
mock.Mock(return_value=[]))
self.mock_object(configparser, 'ConfigParser',
mock.Mock(return_value=parser_mock))
self.mock_object(os, 'listdir',
mock.Mock(return_value=fakes.FAKE_DIR_FILES))
self.mock_object(os, 'access', mock.Mock(return_value=True))
config_parser, file_path = yum_config._read_config_file(
fakes.FAKE_SECTION1)
self.assertEqual(parser_mock, config_parser)
self.assertIsNone(file_path)
self.assertEqual(expected_dir_path, result)
@mock.patch('builtins.open')
def test_update_section(self, open):
yum_config = self._create_yum_config_obj(
file_path=fakes.FAKE_FILE_PATH,
valid_options=fakes.FAKE_SUPP_OPTIONS)
config_parser = fakes.FakeConfigParser({fakes.FAKE_SECTION1: {}})
@ -186,13 +139,14 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase):
updates = {fakes.FAKE_OPTION1: 'new_fake_value'}
yum_config.update_section(fakes.FAKE_SECTION1, updates)
yum_config.update_section(fakes.FAKE_SECTION1, updates,
file_path=fakes.FAKE_FILE_PATH)
mock_read_config.assert_called_once_with(fakes.FAKE_SECTION1)
mock_read_config.assert_called_once_with(fakes.FAKE_FILE_PATH,
section=fakes.FAKE_SECTION1)
def test_update_section_invalid_options(self):
yum_config = self._create_yum_config_obj(
file_path=fakes.FAKE_FILE_PATH,
valid_options=fakes.FAKE_SUPP_OPTIONS)
updates = {'invalid_option': 'new_fake_value'}
@ -200,15 +154,15 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase):
self.assertRaises(exc.TripleOYumConfigInvalidOption,
yum_config.update_section,
fakes.FAKE_SECTION1,
updates)
updates,
file_path=fakes.FAKE_FILE_PATH)
def test_update_section_file_not_found(self):
yum_config = self._create_yum_config_obj(
file_path=fakes.FAKE_FILE_PATH,
valid_options=fakes.FAKE_SUPP_OPTIONS)
mock_read_config = self.mock_object(
yum_config, '_read_config_file',
mock.Mock(return_value=(mock.Mock(), None)))
mock_get_configs = self.mock_object(
yum_config, '_get_config_files',
mock.Mock(return_value=[]))
updates = {fakes.FAKE_OPTION1: 'new_fake_value'}
@ -216,8 +170,7 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase):
yum_config.update_section,
fakes.FAKE_SECTION1,
updates)
mock_read_config.assert_called_once_with(fakes.FAKE_SECTION1)
mock_get_configs.assert_called_once_with(fakes.FAKE_SECTION1)
@ddt.ddt
@ -229,8 +182,7 @@ class TestTripleOYumRepoConfig(test_main.TestTripleoYumConfigBase):
self.mock_object(os.path, 'isfile')
self.mock_object(os, 'access')
self.mock_object(os.path, 'isdir')
cfg_obj = yum_cfg.TripleOYumRepoConfig(
file_path=fakes.FAKE_FILE_PATH)
cfg_obj = yum_cfg.TripleOYumRepoConfig()
mock_update = self.mock_object(yum_cfg.TripleOYumConfig,
'update_section')
@ -240,10 +192,12 @@ class TestTripleOYumRepoConfig(test_main.TestTripleoYumConfigBase):
if enable is not None:
expected_updates['enabled'] = '1' if enable else '0'
cfg_obj.update_section(fakes.FAKE_SECTION1, updates, enable=enable)
cfg_obj.update_section(fakes.FAKE_SECTION1, set_dict=updates,
file_path=fakes.FAKE_FILE_PATH, enabled=enable)
mock_update.assert_called_once_with(fakes.FAKE_SECTION1,
expected_updates)
expected_updates,
file_path=fakes.FAKE_FILE_PATH)
@ddt.ddt