From 9bc5d12529946f784acfbd39e79f2478494c337f Mon Sep 17 00:00:00 2001 From: Douglas Viroel Date: Wed, 4 Aug 2021 11:33:26 -0300 Subject: [PATCH] Add module to generate CentOS Compose repos This patch extends yum-config funcionality to provide a module that generates compose repos based on a compose URL. Co-authored-by: Bhagyashri Shewale Change-Id: I9f8dbede85e1c60de3a86991abcd53fc86444155 Signed-off-by: Douglas Viroel --- docs/yum_config.md | 17 +- molecule/default/converge.yml | 33 ++- .../tripleo_repos/yum_config/__main__.py | 80 +++++- .../tripleo_repos/yum_config/compose_repos.py | 202 ++++++++++++++ .../tripleo_repos/yum_config/constants.py | 29 ++ .../tripleo_repos/yum_config/exceptions.py | 7 + .../tripleo_repos/yum_config/yum_config.py | 256 +++++++++++------- plugins/modules/yum_config.py | 93 ++++++- tests/sanity/ignore.txt | 1 + tests/unit/yum_config/test_main.py | 9 +- tests/unit/yum_config/test_yum_config.py | 126 +++------ 11 files changed, 653 insertions(+), 200 deletions(-) create mode 100644 plugins/module_utils/tripleo_repos/yum_config/compose_repos.py diff --git a/docs/yum_config.md b/docs/yum_config.md index 36d88d5..9ee0bd3 100644 --- a/docs/yum_config.md +++ b/docs/yum_config.md @@ -15,8 +15,8 @@ its repository and invoking in command line: This subcommand lets you enable or disable a repo and sets its configuration options. The *tripleo-yum-config* module will search for the provided repo name in all *.repo* files at REPO_DIR_PATH. - Optionally, you can provide a dir path where your repo files live or specify the full path of the repo file. - By default REPO_DIR_PATH is set to */etc/yum.repod.d/*. + Optionally, you can provide a path where your repo files live or specify the full path of the repo file. + By default REPO_DIR_PATH is set to */etc/yum.repos.d/*. Examples: ``` @@ -48,6 +48,19 @@ its repository and invoking in command line: ``` sudo python -m tripleo_yum_config global --set-opts keepcache=1 cachedir="/var/cache/dnf" ``` + +* **enable-compose-repos** + + This subcommand will enable a list os yum repos based on the metadata retrieved from the `compose-url`. + The *tripleo-yum-config* module will create new repo files at REPO_DIR_PATH and enable them. + Optionally, you can provide a path where your repo files live, specify the variants that should be created and which repos need to be disabled afterwards. + By default REPO_DIR_PATH is set to */etc/yum.repos.d/*. + + Example: + ``` + sudo python -m tripleo_yum_config enable-compose-repos --compose-url https://composes.centos.org/latest-CentOS-Stream-8/compose/ --release centos-stream-8 --disable-all-conflicting + ``` + #### Install using setup.py Installation using python setup.py requires sudo, because the python source diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index 6c345f4..d5b6a43 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -2,8 +2,8 @@ - name: Converge hosts: all tasks: - - - name: "Test yum_config repo" + - name: "Test yum_config repo config" + become: true tripleo.repos.yum_config: type: repo name: appstream @@ -13,12 +13,41 @@ # the line below which disables molecule idemptotence test. - molecule-idempotence-notest + - name: "Test yum_config global config" + become: true + tripleo.repos.yum_config: + type: global + file_path: /etc/dnf/dnf.conf + set_options: + skip_if_unavailable: "False" + keepcache: "0" + tags: + # TODO: fix yum_config to correctly report changed state and uncomment + # the line below which disables molecule idemptotence test. + - molecule-idempotence-notest + - name: "Check get_hash" tripleo.repos.get_hash: release: master + - name: "Check get_hash with invalid url" tripleo.repos.get_hash: release: master dlrn_url: 'https://httpbin.org/status/404' register: result failed_when: result is success + + - name: "Test yum_config enable-compose-repos" + become: true + tripleo.repos.yum_config: + type: enable-compose-repos + compose_url: https://composes.centos.org/latest-CentOS-Stream-8/compose/ + centos_release: centos-stream-8 + variants: + - AppStream + - BaseOS + disable_repos: + - /etc/yum.repos.d/CentOS-Stream-AppStream.repo + - /etc/yum.repos.d/CentOS-Stream-BaseOS.repo + tags: + - molecule-idempotence-notest diff --git a/plugins/module_utils/tripleo_repos/yum_config/__main__.py b/plugins/module_utils/tripleo_repos/yum_config/__main__.py index 1e0d9cf..a3c66f3 100644 --- a/plugins/module_utils/tripleo_repos/yum_config/__main__.py +++ b/plugins/module_utils/tripleo_repos/yum_config/__main__.py @@ -17,8 +17,10 @@ import argparse import logging import sys -import tripleo_repos.yum_config.yum_config as cfg +import tripleo_repos.yum_config.compose_repos as compose_repos +import tripleo_repos.yum_config.constants as const import tripleo_repos.yum_config.dnf_manager as dnf_mgr +import tripleo_repos.yum_config.yum_config as cfg def options_to_dict(options): @@ -63,8 +65,9 @@ def main(): repo_args_parser.add_argument( '--config-dir-path', dest='config_dir_path', + default=const.YUM_REPO_DIR, help=( - 'set the absolute directory path that holds all repo or module ' + 'set the absolute directory path that holds all repo ' 'configuration files') ) @@ -98,13 +101,61 @@ def main(): help="sets module profile" ) + # Compose repo arguments + compose_args_parser = argparse.ArgumentParser(add_help=False) + compose_args_parser.add_argument( + '--compose-url', + dest='compose_url', + required=True, + help='CentOS compose URL' + ) + compose_args_parser.add_argument( + '--release', + dest='release', + choices=const.COMPOSE_REPOS_RELEASES, + default='centos-stream-8', + help='target CentOS release.' + ) + compose_args_parser.add_argument( + '--arch', + choices=const.COMPOSE_REPOS_SUPPORTED_ARCHS, + default='x86_64', + help='set the architecture for the destination repos.' + ) + compose_args_parser.add_argument( + '--disable-repos', + nargs='+', + help='list of repo names or repo absolute file paths to be disabled.' + ) + compose_args_parser.add_argument( + '--disable-all-conflicting', + action='store_true', + dest='disable_conflicting', + default=False, + help='after enabling compose repos, disable all other repos that ' + 'match variant names.' + ) + compose_args_parser.add_argument( + '--variants', + nargs='+', + help='Name of the repos to be enabled. Default behavior is to enable ' + 'all that match a specific release and architecture.' + ) + compose_args_parser.add_argument( + '--config-dir-path', + dest='config_dir_path', + default=const.YUM_REPO_DIR, + help='set the absolute directory path that holds all repo ' + 'configuration files' + ) + # Common file path argument common_parse = argparse.ArgumentParser(add_help=False) common_parse.add_argument( '--config-file-path', dest='config_file_path', help=('set the absolute file path of the configuration file to be ' - 'updated') + 'updated.') ) # Main parser @@ -133,6 +184,11 @@ def main(): parents=[common_parse, options_parse], help='updates global yum configuration options' ) + subparsers.add_parser( + 'enable-compose-repos', + parents=[compose_args_parser], + help='enable CentOS compose repos based on an compose url.' + ) args = main_parser.parse_args() if args.command is None: @@ -146,10 +202,11 @@ def main(): if args.command == 'repo': set_dict = options_to_dict(args.set_opts) config_obj = cfg.TripleOYumRepoConfig( - file_path=args.config_file_path, dir_path=args.config_dir_path) - config_obj.update_section(args.name, set_dict, enable=args.enable) + config_obj.update_section(args.name, set_dict, + file_path=args.config_file_path, + enabled=args.enable) elif args.command == 'module': dnf_mod_mgr = dnf_mgr.DnfModuleManager() @@ -163,6 +220,19 @@ def main(): config_obj.update_section('main', set_dict) + elif args.command == 'enable-compose-repos': + repo_obj = compose_repos.TripleOYumComposeRepoConfig( + args.compose_url, + args.release, + dir_path=args.config_dir_path, + arch=args.arch) + + repo_obj.enable_compose_repos(variants=args.variants, + override_repos=args.disable_conflicting) + if args.disable_repos: + for file in args.disable_repos: + repo_obj.update_all_sections(file, enabled=False) + def cli_entrypoint(): try: diff --git a/plugins/module_utils/tripleo_repos/yum_config/compose_repos.py b/plugins/module_utils/tripleo_repos/yum_config/compose_repos.py new file mode 100644 index 0000000..c9b1359 --- /dev/null +++ b/plugins/module_utils/tripleo_repos/yum_config/compose_repos.py @@ -0,0 +1,202 @@ +# Copyright 2021 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +from __future__ import (absolute_import, division, print_function) + +import logging +import json +import os +import re +import urllib.parse +import urllib.request + +from .constants import ( + YUM_REPO_DIR, + YUM_REPO_FILE_EXTENSION, + YUM_REPO_SUPPORTED_OPTIONS, + COMPOSE_REPOS_RELEASES, + COMPOSE_REPOS_INFO_PATH, + COMPOSE_REPOS_URL_PATTERN, + COMPOSE_REPOS_URL_REPLACE_STR, +) +from .exceptions import ( + TripleOYumConfigInvalidSection, + TripleOYumConfigComposeError, +) +from .yum_config import ( + TripleOYumConfig +) + +__metaclass__ = type + + +class TripleOYumComposeRepoConfig(TripleOYumConfig): + """Manages yum repo configuration files for CentOS Compose.""" + + def __init__(self, compose_url, release, dir_path=None, arch=None): + conf_dir_path = dir_path or YUM_REPO_DIR + self.arch = arch or 'x86_64' + + # 1. validate release name + if release not in COMPOSE_REPOS_RELEASES: + msg = 'CentOS release not supported.' + raise TripleOYumConfigComposeError(error_msg=msg) + self.release = release + + # 2. Validate URL + pattern = re.compile(COMPOSE_REPOS_URL_PATTERN[self.release]) + if not pattern.match(compose_url): + msg = 'The provided URL does not match the expect pattern.' + raise TripleOYumConfigComposeError(error_msg=msg) + + # 3. Get compose info from url + segments = [compose_url, + COMPOSE_REPOS_INFO_PATH[self.release]] + self.compose_info_url = '/'.join(s.strip('/') for s in segments) + self.compose_info = self._get_compose_info() + + # 4. Get compose-id from metadata + self.compose_id = self.compose_info['compose']['id'] + + # 5. Replace the compose-id from url to avoid 'labels' + repl_args = {'compose_id': self.compose_id} + self.compose_url = ( + pattern.sub( + COMPOSE_REPOS_URL_REPLACE_STR[self.release] % repl_args, + compose_url) + ) + + super(TripleOYumComposeRepoConfig, self).__init__( + valid_options=YUM_REPO_SUPPORTED_OPTIONS, + dir_path=conf_dir_path, + file_extension=YUM_REPO_FILE_EXTENSION) + + def _get_compose_info(self): + """Retrieve compose info for a provided compose-id url.""" + # NOTE(dviroel): works for both centos 8 and 9 + try: + logging.debug("Retrieving compose info from url: %s", + self.compose_info_url) + res = urllib.request.urlopen(self.compose_info_url) + except Exception: + msg = ("Failed to retrieve compose info from url: %s" + % self.compose_info_url) + raise TripleOYumConfigComposeError(error_msg=msg) + compose_info = json.loads(res.read()) + if compose_info['header']['version'] != "1.2": + # NOTE(dviroel): Log a warning just in case we receive a different + # version here. Code may fail depending on the change. + logging.warning("Expecting compose info version '1.2' but got %s.", + compose_info['header']['version']) + return compose_info['payload'] + + def _get_repo_name(self, variant): + return " ".join([self.compose_id, variant]) + + def _get_repo_filename(self, variant): + return "-".join([self.compose_id, variant]) + '.repo' + + def _get_repo_base_url(self, variant): + """Build the base_url based on variant name and system architecture.""" + variant_info = self.compose_info['variants'][variant] + if not variant_info['paths'].get('repository', {}).get(self.arch): + # Variant has no support yet + return None + segments = [self.compose_url, + variant_info['paths']['repository'][self.arch]] + return '/'.join(s.strip('/') for s in segments) + + def get_compose_variants(self): + return self.compose_info['variants'].keys() + + def enable_compose_repos(self, variants=None, override_repos=False): + """Enable CentOS compose repos of a given variant list. + + This function will build from scratch all repos for a given compose-id + url. If a list of variants is not provided, it will enable all for all + variants returned from compose info. + + :param variants: A list of variant names to be enabled. + :param override_repos: True if all matching variants in the same + repo directory should be disable in favor of the new repos. + """ + if variants: + for var in variants: + if not (var in self.compose_info['variants'].keys()): + msg = 'One or more provided variants are invalid.' + raise TripleOYumConfigComposeError(error_msg=msg) + + else: + variants = self.compose_info['variants'].keys() + + updated_repos = {} + for var in variants: + base_url = self._get_repo_base_url(var) + if not base_url: + continue + add_dict = { + 'name': self._get_repo_name(var), + 'baseurl': base_url, + 'enabled': '1', + 'gpgcheck': '0', + } + filename = self._get_repo_filename(var) + file_path = os.path.join(self.dir_path, filename) + # create a file if doesn't exist and add a section to it + try: + self.add_section(var.lower(), add_dict, file_path) + except TripleOYumConfigInvalidSection: + logging.debug("Section '%s' that already exists in this file. " + "Skipping...", var) + # needed to override other repos + updated_repos[var.lower()] = file_path + + if override_repos: + for var in updated_repos: + config_files = self._get_config_files(var) + for file in config_files: + if file != updated_repos[var]: + msg = ("Disabling matching section '%(section)s' in " + "configuration file: %(file)s.") + msg_args = { + 'section': var, + 'file': file, + } + logging.debug(msg, msg_args) + self.update_section(var, enabled=False, file_path=file) + + def add_section(self, section, add_dict, file_path): + # Create a new file if it does not exists + if not os.path.isfile(file_path): + with open(file_path, '+w'): + pass + super(TripleOYumComposeRepoConfig, self).add_section( + section, add_dict, file_path) + + def update_section( + self, section, set_dict=None, enabled=None, file_path=None): + update_dict = set_dict or {} + if enabled is not None: + update_dict['enabled'] = '1' if enabled else '0' + if update_dict: + super(TripleOYumComposeRepoConfig, self).update_section( + section, update_dict, file_path=file_path) + + def update_all_sections(self, file_path, set_dict=None, enabled=None): + update_dict = set_dict or {} + if enabled is not None: + update_dict['enabled'] = '1' if enabled else '0' + if update_dict: + super(TripleOYumComposeRepoConfig, self).update_all_sections( + update_dict, file_path) diff --git a/plugins/module_utils/tripleo_repos/yum_config/constants.py b/plugins/module_utils/tripleo_repos/yum_config/constants.py index c55e3af..0e322b0 100644 --- a/plugins/module_utils/tripleo_repos/yum_config/constants.py +++ b/plugins/module_utils/tripleo_repos/yum_config/constants.py @@ -39,3 +39,32 @@ YUM_REPO_FILE_EXTENSION = '.repo' Default constants for yum/dnf global configurations. """ YUM_GLOBAL_CONFIG_FILE_PATH = '/etc/yum.conf' + +""" +CentOS Stream compose repos defaults +""" +COMPOSE_REPOS_RELEASES = [ + "centos-stream-8", + "centos-stream-9" +] + +COMPOSE_REPOS_SUPPORTED_ARCHS = [ + "aarch64", + "ppc64le", + "x86_64" +] + +COMPOSE_REPOS_URL_PATTERN = { + "centos-stream-8": r"(^https:.*.centos.org/)([^/]*)(/compose/?$)", + "centos-stream-9": r"(^https:.*.centos.org/.*/)(.*)(/compose/?$)", +} + +COMPOSE_REPOS_URL_REPLACE_STR = { + "centos-stream-8": r"\1%(compose_id)s\3", + "centos-stream-9": r"\1%(compose_id)s\3", +} + +COMPOSE_REPOS_INFO_PATH = { + "centos-stream-8": "metadata/composeinfo.json", + "centos-stream-9": "metadata/composeinfo.json", +} diff --git a/plugins/module_utils/tripleo_repos/yum_config/exceptions.py b/plugins/module_utils/tripleo_repos/yum_config/exceptions.py index c682ab4..41a8b57 100644 --- a/plugins/module_utils/tripleo_repos/yum_config/exceptions.py +++ b/plugins/module_utils/tripleo_repos/yum_config/exceptions.py @@ -60,3 +60,10 @@ class TripleOYumConfigInvalidOption(Base): def __init__(self, error_msg): super(TripleOYumConfigInvalidOption, self).__init__(error_msg) + + +class TripleOYumConfigComposeError(Base): + """An error occurred while configuring CentOS compose repos.""" + + def __init__(self, error_msg): + super(TripleOYumConfigComposeError, self).__init__(error_msg) diff --git a/plugins/module_utils/tripleo_repos/yum_config/yum_config.py b/plugins/module_utils/tripleo_repos/yum_config/yum_config.py index ac48f92..1c338ff 100644 --- a/plugins/module_utils/tripleo_repos/yum_config/yum_config.py +++ b/plugins/module_utils/tripleo_repos/yum_config/yum_config.py @@ -31,13 +31,18 @@ from .exceptions import ( TripleOYumConfigInvalidOption, TripleOYumConfigInvalidSection, TripleOYumConfigNotFound, - TripleOYumConfigPermissionDenied, ) __metaclass__ = type +def validated_file_path(file_path): + if os.path.isfile(file_path) and os.access(file_path, os.W_OK): + return True + return False + + class TripleOYumConfig: """ This class is a base class for updating yum configuration files in @@ -74,171 +79,230 @@ class TripleOYumConfig: logger.addHandler(handler) logger.setLevel(logging.INFO) - def __init__(self, valid_options=None, file_path=None, dir_path=None, - file_extension=None): + def __init__(self, valid_options=None, dir_path=None, file_extension=None): """ Creates a TripleOYumConfig object that holds configuration file information. :param valid_options: A list of options that can be updated on this file. - :param file_path: The file path to configuration file to be updated. :param dir_path: The directory path that this class can use to search for configuration files to be updated. :param: file_extension: File extension to filter configuration files in the search directory. """ - self.config_file_path = file_path self.dir_path = dir_path self.file_extension = file_extension self.valid_options = valid_options # Sanity checks - if not (file_path or dir_path): - msg = ('A configuration file path or a directory path must be ' - 'provided.') - raise TripleOYumConfigNotFound(error_msg=msg) - - if file_path: - if not os.path.isfile(file_path): - msg = ('The configuration file "{0}" was not found in the ' - 'provided path.').format(file_path) - raise TripleOYumConfigNotFound(error_msg=msg) - if not os.access(file_path, os.W_OK): - msg = ('The configuration file {0} is not ' - 'writable.'.format(file_path)) - raise TripleOYumConfigPermissionDenied(error_msg=msg) - if dir_path: if not os.path.isdir(dir_path): msg = ('The configuration dir "{0}" was not found in the ' 'provided path.').format(dir_path) raise TripleOYumConfigNotFound(error_msg=msg) - def _read_config_file(self, section): - """Read the configuration file associate with this object. + def _read_config_file(self, file_path, section=None): + """Reads a configuration file. - If no configuration file is provided, this method will search for - 'section' name in all files inside the configuration directory. The - first occurrence of 'section' will be returned, and a warning will be - logged if more than one configuration file has the same 'section' set. - - :param section: The name of the section to be updated. - :return: a config parser object and the file path. + :param section: The name of the section that will be update. Only used + to fail earlier if the section is not found. + :return: a config parser object and the full file path. """ config = configparser.ConfigParser() - # A) A configuration file path was provided. - if self.config_file_path: - try: - config.read(self.config_file_path) - except configparser.Error: - msg = 'Unable to parse configuration file {0}.'.format( - self.config_file_path) - raise TripleOYumConfigFileParseError(error_msg=msg) + file_paths = [file_path] + if self.dir_path: + # if dir_path is configured, we can search for filename there + file_paths.append(os.path.join(self.dir_path, file_path)) - if section not in config.sections(): - msg = ('The provided section "{0}" was not found in the ' - 'configuration file {1}.').format( - section, self.config_file_path) - raise TripleOYumConfigInvalidSection(error_msg=msg) + valid_file_path = None + for file in file_paths: + if validated_file_path(file): + valid_file_path = file + break + if not valid_file_path: + msg = ('The configuration file "{0}" was ' + 'not found.'.format(file_path)) + raise TripleOYumConfigNotFound(error_msg=msg) - return config, self.config_file_path + try: + config.read(valid_file_path) + except configparser.Error: + msg = 'Unable to parse configuration file {0}.'.format( + valid_file_path) + raise TripleOYumConfigFileParseError(error_msg=msg) - # B) Search for a configuration file that has the provided section - section_found = False - config_file_path = None - for file in os.listdir(self.dir_path): - # Skip files that don't match the file extension or are not - # writable - if self.file_extension and not file.endswith( - self.file_extension): - continue - if not os.access(os.path.join(self.dir_path, file), os.W_OK): - continue + if section and section not in config.sections(): + msg = ('The provided section "{0}" was not found in the ' + 'configuration file {1}.').format( + section, valid_file_path) + raise TripleOYumConfigInvalidSection(error_msg=msg) - tmp_config = configparser.ConfigParser() - try: - tmp_config.read(os.path.join(self.dir_path, file)) - except configparser.Error: - continue - if section in tmp_config.sections(): - if section_found: - logging.warning('Section "%s" is listed more than once in ' - 'configuration files.', section) - else: - # Read the first occurrence of 'section' - config_file_path = os.path.join(self.dir_path, file) - config.read(config_file_path) - section_found = True + return config, valid_file_path - return config, config_file_path + def _get_config_files(self, section): + """Gets all configuration file paths for a given section. - def update_section(self, section, set_dict): - """Updates a set of options for a specified section. + This method will search for a 'section' name in all files inside the + configuration directory. All files with 'section' will be returned. + + :param section: Section to be found inside configuration files. + :return: A list of config file paths. + """ + # Search for a configuration file that has the provided section + config_files_path = [] + if section and self.dir_path: + for file in os.listdir(self.dir_path): + # Skip files that don't match the file extension or are not + # writable + if self.file_extension and not file.endswith( + self.file_extension): + continue + if not os.access(os.path.join(self.dir_path, file), os.W_OK): + continue + + tmp_config = configparser.ConfigParser() + try: + tmp_config.read(os.path.join(self.dir_path, file)) + except configparser.Error: + continue + if section in tmp_config.sections(): + config_files_path.append(os.path.join(self.dir_path, file)) + + return config_files_path + + def update_section(self, section, set_dict, file_path=None): + """Updates a set of options of a section. + + If a file path is not provided by the caller, this function will search + for the section in all files located in the working directory and + update each one of them. :param section: Name of the section on the configuration file that will be updated. :param set_dict: Dict with all options and values to be updated in the configuration file section. + :param file_path: Path to the configuration file to be updated. """ if self.valid_options: if not all(key in self.valid_options for key in set_dict.keys()): msg = 'One or more provided options are not valid.' raise TripleOYumConfigInvalidOption(error_msg=msg) - config, config_file_path = self._read_config_file(section) - if not (config and config_file_path): - msg = ('The provided section "{0}" was not found within any ' - 'configuration file.').format(section) + files = [file_path] if file_path else self._get_config_files(section) + if not files: + msg = ('No configuration files were found for the provided ' + 'section {0}'.format(section)) raise TripleOYumConfigNotFound(error_msg=msg) - # Update configuration file with dict updates - config[section].update(set_dict) + for file in files: + config, file = self._read_config_file(file, section=section) + # Update configuration file with dict updates + config[section].update(set_dict) + with open(file, 'w') as f: + config.write(f) - with open(config_file_path, 'w') as file: + logging.info("Section '%s' was successfully " + "updated.", section) + + def add_section(self, section, add_dict, file_path): + """ Adds a new section with options in a provided config file. + + :param section: Section name to be added to the config file. + :param add_dict: Dict with all options and values to be added into the + new section. + :param file_path: Path to the configuration file to be updated. + """ + # This section shouldn't exist in the provided file + config, file_path = self._read_config_file(file_path=file_path) + if section in config.sections(): + msg = ("Section '%s' already exists in the configuration " + "file.", section) + raise TripleOYumConfigInvalidSection(error_msg=msg) + + # Add new section + config.add_section(section) + # Update configuration file with dict updates + config[section].update(add_dict) + + with open(file_path, '+w') as file: config.write(file) - logging.info("Section '%s' was successfully updated.", section) + logging.info("Section '%s' was successfully " + "added.", section) + + def update_all_sections(self, set_dict, file_path): + """Updates all section of a given configuration file. + + :param set_dict: Dict with all options and values to be updated in + the configuration file. + :param file_path: Path to the configuration file to be updated. + """ + if self.valid_options: + if not all(key in self.valid_options for key in set_dict.keys()): + msg = 'One or more provided options are not valid.' + raise TripleOYumConfigInvalidOption(error_msg=msg) + + config, file_path = self._read_config_file(file_path) + for section in config.sections(): + config[section].update(set_dict) + + with open(file_path, '+w') as file: + config.write(file) + + logging.info("All sections for '%s' were successfully " + "updated.", file_path) class TripleOYumRepoConfig(TripleOYumConfig): """Manages yum repo configuration files.""" - def __init__(self, file_path=None, dir_path=None): - if file_path: - logging.info( - "Using '%s' as yum repo configuration file.", file_path) + def __init__(self, dir_path=None): conf_dir_path = dir_path or YUM_REPO_DIR super(TripleOYumRepoConfig, self).__init__( valid_options=YUM_REPO_SUPPORTED_OPTIONS, - file_path=file_path, dir_path=conf_dir_path, file_extension=YUM_REPO_FILE_EXTENSION) - def update_section(self, section, set_dict, enable=None): - if enable is not None: - set_dict['enabled'] = '1' if enable else '0' - - super(TripleOYumRepoConfig, self).update_section(section, set_dict) + def update_section( + self, section, set_dict=None, file_path=None, enabled=None): + update_dict = set_dict or {} + if enabled is not None: + update_dict['enabled'] = '1' if enabled else '0' + if update_dict: + super(TripleOYumRepoConfig, self).update_section( + section, update_dict, file_path=file_path) class TripleOYumGlobalConfig(TripleOYumConfig): """Manages yum global configuration file.""" def __init__(self, file_path=None): - conf_file_path = file_path or YUM_GLOBAL_CONFIG_FILE_PATH + self.conf_file_path = file_path or YUM_GLOBAL_CONFIG_FILE_PATH logging.info("Using '%s' as yum global configuration " - "file.", conf_file_path) - if file_path is None: + "file.", self.conf_file_path) + if file_path is not None: + # validate user provided file path + validated_file_path(file_path) + else: # If there is no default 'yum.conf' configuration file, we need to # create it. If the user specify another conf file that doesn't # exists, the operation will fail. - if not os.path.isfile(conf_file_path): + if not os.path.isfile(self.conf_file_path): config = configparser.ConfigParser() - config.read(conf_file_path) + config.read(self.conf_file_path) config.add_section('main') - with open(conf_file_path, '+w') as file: + with open(self.conf_file_path, '+w') as file: config.write(file) - super(TripleOYumGlobalConfig, self).__init__(file_path=conf_file_path) + super(TripleOYumGlobalConfig, self).__init__() + + def update_section(self, section, set_dict, file_path=None): + super(TripleOYumGlobalConfig, self).update_section( + section, set_dict, file_path=(file_path or self.conf_file_path)) + + def add_section(self, section, set_dict, file_path=None): + add_file_path = file_path or self.conf_file_path + super(TripleOYumGlobalConfig, self).add_section( + section, set_dict, add_file_path) diff --git a/plugins/modules/yum_config.py b/plugins/modules/yum_config.py index c8108b4..993f493 100644 --- a/plugins/modules/yum_config.py +++ b/plugins/modules/yum_config.py @@ -24,7 +24,7 @@ options: - The type of yum configuration to be changed. required: true type: str - choices: [repo, module, global] + choices: [repo, module, global, 'enable-compose-repos'] name: description: - Name of the repo or module to be changed. This options is @@ -65,6 +65,40 @@ options: - Absolute path of the directory that contains the configuration file to be changed. type: path + default: /etc/yum.repos.d + compose_url: + description: + - URL that contains CentOS compose repositories. + type: str + centos_release: + description: + - Target CentOS release. + type: str + choices: [centos-stream-8, centos-stream-9] + arch: + description: + - System architecture which the repos will be configure. + type: str + choices: [aarch64, ppc64le, x86_64] + default: x86_64 + variants: + description: + - Repository variants that should be configured. If not provided, + all available variants will be configured. + type: list + elements: str + disable_conflicting_variants: + description: + - Disable all repos from the same directory that match variants' + name. + type: bool + default: false + disable_repos: + description: + - List with file path of repos that should be disabled after + successfully enabling all compose repos. + type: list + elements: str author: - Douglas Viroel (@viroel) @@ -113,6 +147,20 @@ EXAMPLES = r''' set_options: skip_if_unavailable: "False" keepcache: "0" + +- name: Configure a set of repos based on latest CentOS Stream 8 compose + become: true + become_user: root + tripleo_yup_config: + compose_url: https://composes.centos.org/latest-CentOS-Stream-8/compose/ + centos_release: centos-stream-8 + variants: + - AppStream + - BaseOS + disable_conflicting_variants: true + disable_repos: + - /etc/yum.repos.d/CentOS-Linux-AppStream.repo + - /etc/yum.repos.d/CentOS-Linux-BaseOS.repo ''' RETURN = r''' # ''' @@ -121,8 +169,14 @@ from ansible.module_utils.basic import AnsibleModule # noqa: E402 def run_module(): + try: + import ansible_collections.tripleo.repos.plugins.module_utils. \ + tripleo_repos.yum_config.constants as const + except ImportError: + import tripleo_repos.yum_config.constants as const # define available arguments/parameters a user can pass to the module - supported_config_types = ['repo', 'module', 'global'] + supported_config_types = ['repo', 'module', 'global', + 'enable-compose-repos'] supported_module_operations = ['install', 'remove', 'reset'] module_args = dict( type=dict(type='str', required=True, choices=supported_config_types), @@ -133,7 +187,17 @@ def run_module(): profile=dict(type='str'), set_options=dict(type='dict', default={}), file_path=dict(type='path'), - dir_path=dict(type='path'), + dir_path=dict(type='path', default=const.YUM_REPO_DIR), + compose_url=dict(type='str'), + centos_release=dict(type='str', + choices=const.COMPOSE_REPOS_RELEASES), + arch=dict(type='str', choices=const.COMPOSE_REPOS_SUPPORTED_ARCHS, + default='x86_64'), + variants=dict(type='list', default=[], + elements='str'), + disable_conflicting_variants=dict(type='bool', default=False), + disable_repos=dict(type='list', default=[], + elements='str'), ) module = AnsibleModule( @@ -141,6 +205,7 @@ def run_module(): required_if=[ ["type", "repo", ["name"]], ["type", "module", ["name"]], + ["type", "enable-compose-repos", ["compose_url"]], ], supports_check_mode=False ) @@ -162,18 +227,36 @@ def run_module(): tripleo_repos.yum_config.dnf_manager as dnf_mgr import ansible_collections.tripleo.repos.plugins.module_utils.\ tripleo_repos.yum_config.yum_config as cfg + import ansible_collections.tripleo.repos.plugins.module_utils. \ + tripleo_repos.yum_config.compose_repos as repos except ImportError: import tripleo_repos.yum_config.dnf_manager as dnf_mgr import tripleo_repos.yum_config.yum_config as cfg + import tripleo_repos.yum_config.compose_repos as repos if module.params['type'] == 'repo': config_obj = cfg.TripleOYumRepoConfig( - file_path=module.params['file_path'], dir_path=module.params['dir_path']) config_obj.update_section( module.params['name'], m_set_opts, - enable=module.params['enabled']) + file_path=module.params['file_path'], + enabled=module.params['enabled']) + + elif module.params['type'] == 'enable-compose-repos': + # 1. Create compose repo config object + repo_obj = repos.TripleOYumComposeRepoConfig( + module.params['compose_url'], + module.params['centos_release'], + dir_path=module.params['dir_path'], + arch=module.params['arch']) + # 2. enable CentOS compose repos + repo_obj.enable_compose_repos( + variants=module.params['variants'], + override_repos=module.params['disable_conflicting_variants']) + # 3. Disable all repos provided in disable_repos + for file in module.params['disable_repos']: + repo_obj.update_all_sections(file, enabled=False) elif module.params['type'] == 'module': dnf_mod_mgr = dnf_mgr.DnfModuleManager() diff --git a/tests/sanity/ignore.txt b/tests/sanity/ignore.txt index 234bdb9..4607a25 100644 --- a/tests/sanity/ignore.txt +++ b/tests/sanity/ignore.txt @@ -1,2 +1,3 @@ plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py replace-urlopen plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py pylint:ansible-bad-import +plugins/module_utils/tripleo_repos/yum_config/compose_repos.py replace-urlopen diff --git a/tests/unit/yum_config/test_main.py b/tests/unit/yum_config/test_main.py index 73c3dbc..dcee54d 100644 --- a/tests/unit/yum_config/test_main.py +++ b/tests/unit/yum_config/test_main.py @@ -20,8 +20,9 @@ from unittest import mock from . import fakes from . import mock_modules # noqa: F401 import tripleo_repos.yum_config.__main__ as main -import tripleo_repos.yum_config.yum_config as yum_cfg +import tripleo_repos.yum_config.constants as const import tripleo_repos.yum_config.dnf_manager as dnf_mgr +import tripleo_repos.yum_config.yum_config as yum_cfg class TestTripleoYumConfigBase(unittest.TestCase): @@ -56,10 +57,10 @@ class TestTripleoYumConfigMain(TestTripleoYumConfigBase): main.main() expected_dict = {'key1': 'value1', 'key2': 'value2'} - mock_yum_repo_obj.assert_called_once_with( - file_path=fakes.FAKE_FILE_PATH, dir_path=None) + mock_yum_repo_obj.assert_called_once_with(dir_path=const.YUM_REPO_DIR) mock_update_section.assert_called_once_with( - 'fake_repo', expected_dict, enable=True) + 'fake_repo', expected_dict, file_path=fakes.FAKE_FILE_PATH, + enabled=True) @ddt.data('enable', 'disable', 'reset', 'install', 'remove') def test_main_module(self, operation): diff --git a/tests/unit/yum_config/test_yum_config.py b/tests/unit/yum_config/test_yum_config.py index 47b0415..638502e 100644 --- a/tests/unit/yum_config/test_yum_config.py +++ b/tests/unit/yum_config/test_yum_config.py @@ -29,51 +29,25 @@ import tripleo_repos.yum_config.yum_config as yum_cfg class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase): """Tests for TripleYumConfig class and its methods.""" - def _create_yum_config_obj(self, file_path=None, dir_path=None, - valid_options=None, file_extension=None): + def _create_yum_config_obj(self, dir_path=None, valid_options=None, + file_extension=None): self.mock_object(os.path, 'isfile') self.mock_object(os, 'access') self.mock_object(os.path, 'isdir') - return yum_cfg.TripleOYumConfig(file_path=file_path, dir_path=dir_path, + return yum_cfg.TripleOYumConfig(dir_path=dir_path, valid_options=valid_options, file_extension=file_extension) - @ddt.data( - {'file_path': None, 'dir_path': None, 'is_file_ret': None, - 'access_ret': None, 'is_dir_ret': None, - 'exception': exc.TripleOYumConfigNotFound}, - - {'file_path': 'fake_path', 'dir_path': None, 'is_file_ret': False, - 'access_ret': None, 'is_dir_ret': None, - 'exception': exc.TripleOYumConfigNotFound}, - - {'file_path': 'fake_path', 'dir_path': None, 'is_file_ret': True, - 'access_ret': False, 'is_dir_ret': None, - 'exception': exc.TripleOYumConfigPermissionDenied}, - - {'file_path': None, 'dir_path': 'fake_dir', 'is_file_ret': None, - 'access_ret': None, 'is_dir_ret': False, - 'exception': exc.TripleOYumConfigNotFound}, - ) - @ddt.unpack - def test_tripleo_yum_config_invalid_parameters( - self, file_path, dir_path, is_file_ret, access_ret, is_dir_ret, - exception): - self.mock_object(os.path, 'isfile', - mock.Mock(return_value=is_file_ret)) - self.mock_object(os, 'access', - mock.Mock(return_value=access_ret)) + def test_tripleo_yum_config_invalid_dir_path(self): self.mock_object(os.path, 'isdir', - mock.Mock(return_value=is_dir_ret)) + mock.Mock(return_value=False)) - self.assertRaises(exception, + self.assertRaises(exc.TripleOYumConfigNotFound, yum_cfg.TripleOYumConfig, - file_path=file_path, - dir_path=dir_path) + dir_path='fake_dir_path') def test_read_config_file_path(self): - yum_config = self._create_yum_config_obj( - file_path=fakes.FAKE_FILE_PATH) + yum_config = self._create_yum_config_obj() parser_mock = mock.Mock() self.mock_object(configparser, 'ConfigParser', @@ -83,7 +57,8 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase): mock.Mock(return_value=fakes.FAKE_SECTIONS)) config_parser, file_path = yum_config._read_config_file( - fakes.FAKE_SECTION1 + fakes.FAKE_FILE_PATH, + section=fakes.FAKE_SECTION1 ) self.assertEqual(parser_mock, config_parser) @@ -91,8 +66,7 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase): read_mock.assert_called_once_with(fakes.FAKE_FILE_PATH) def test_read_config_file_path_parse_error(self): - yum_config = self._create_yum_config_obj( - file_path=fakes.FAKE_FILE_PATH) + yum_config = self._create_yum_config_obj() parser_mock = mock.Mock() self.mock_object(configparser, 'ConfigParser', @@ -102,13 +76,13 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase): self.assertRaises(exc.TripleOYumConfigFileParseError, yum_config._read_config_file, - fakes.FAKE_SECTION1) + fakes.FAKE_FILE_PATH, + section=fakes.FAKE_SECTION1) read_mock.assert_called_once_with(fakes.FAKE_FILE_PATH) def test_read_config_file_path_invalid_section(self): - yum_config = self._create_yum_config_obj( - file_path=fakes.FAKE_FILE_PATH) + yum_config = self._create_yum_config_obj() parser_mock = mock.Mock() self.mock_object(configparser, 'ConfigParser', @@ -119,11 +93,12 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase): self.assertRaises(exc.TripleOYumConfigInvalidSection, yum_config._read_config_file, - 'invalid_section') + fakes.FAKE_FILE_PATH, + section='invalid_section') read_mock.assert_called_once_with(fakes.FAKE_FILE_PATH) - def test_read_config_file_dir(self): + def test_get_config_files(self): yum_config = self._create_yum_config_obj( dir_path=fakes.FAKE_DIR_PATH, file_extension='.conf') @@ -133,50 +108,28 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase): parser_mocks.append(parser_mock) self.mock_object(parser_mock, 'read') - self.mock_object(parser_mocks[1], 'sections', + self.mock_object(parser_mocks[0], 'sections', mock.Mock(return_value=[])) # second file inside dir will have the expected sections - self.mock_object(parser_mocks[2], 'sections', + self.mock_object(parser_mocks[1], 'sections', mock.Mock(return_value=fakes.FAKE_SECTIONS)) + self.mock_object(parser_mocks[2], 'sections', + mock.Mock(return_value=[])) self.mock_object(os, 'listdir', mock.Mock(return_value=fakes.FAKE_DIR_FILES)) self.mock_object(os, 'access', mock.Mock(return_value=True)) self.mock_object(configparser, 'ConfigParser', mock.Mock(side_effect=parser_mocks)) - config_parser, file_path = yum_config._read_config_file( - fakes.FAKE_SECTION1) - expected_dir_path = os.path.join(fakes.FAKE_DIR_PATH, - fakes.FAKE_DIR_FILES[1]) + result = yum_config._get_config_files(fakes.FAKE_SECTION1) + expected_dir_path = [os.path.join(fakes.FAKE_DIR_PATH, + fakes.FAKE_DIR_FILES[1])] - self.assertEqual(parser_mocks[0], config_parser) - self.assertEqual(expected_dir_path, file_path) - - def test_read_config_file_dir_section_not_found(self): - yum_config = self._create_yum_config_obj( - dir_path=fakes.FAKE_DIR_PATH, - file_extension='.conf') - parser_mock = mock.Mock() - self.mock_object(parser_mock, 'read') - self.mock_object(parser_mock, 'sections', - mock.Mock(return_value=[])) - self.mock_object(configparser, 'ConfigParser', - mock.Mock(return_value=parser_mock)) - - self.mock_object(os, 'listdir', - mock.Mock(return_value=fakes.FAKE_DIR_FILES)) - self.mock_object(os, 'access', mock.Mock(return_value=True)) - - config_parser, file_path = yum_config._read_config_file( - fakes.FAKE_SECTION1) - - self.assertEqual(parser_mock, config_parser) - self.assertIsNone(file_path) + self.assertEqual(expected_dir_path, result) @mock.patch('builtins.open') def test_update_section(self, open): yum_config = self._create_yum_config_obj( - file_path=fakes.FAKE_FILE_PATH, valid_options=fakes.FAKE_SUPP_OPTIONS) config_parser = fakes.FakeConfigParser({fakes.FAKE_SECTION1: {}}) @@ -186,13 +139,14 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase): updates = {fakes.FAKE_OPTION1: 'new_fake_value'} - yum_config.update_section(fakes.FAKE_SECTION1, updates) + yum_config.update_section(fakes.FAKE_SECTION1, updates, + file_path=fakes.FAKE_FILE_PATH) - mock_read_config.assert_called_once_with(fakes.FAKE_SECTION1) + mock_read_config.assert_called_once_with(fakes.FAKE_FILE_PATH, + section=fakes.FAKE_SECTION1) def test_update_section_invalid_options(self): yum_config = self._create_yum_config_obj( - file_path=fakes.FAKE_FILE_PATH, valid_options=fakes.FAKE_SUPP_OPTIONS) updates = {'invalid_option': 'new_fake_value'} @@ -200,15 +154,15 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase): self.assertRaises(exc.TripleOYumConfigInvalidOption, yum_config.update_section, fakes.FAKE_SECTION1, - updates) + updates, + file_path=fakes.FAKE_FILE_PATH) def test_update_section_file_not_found(self): yum_config = self._create_yum_config_obj( - file_path=fakes.FAKE_FILE_PATH, valid_options=fakes.FAKE_SUPP_OPTIONS) - mock_read_config = self.mock_object( - yum_config, '_read_config_file', - mock.Mock(return_value=(mock.Mock(), None))) + mock_get_configs = self.mock_object( + yum_config, '_get_config_files', + mock.Mock(return_value=[])) updates = {fakes.FAKE_OPTION1: 'new_fake_value'} @@ -216,8 +170,7 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase): yum_config.update_section, fakes.FAKE_SECTION1, updates) - - mock_read_config.assert_called_once_with(fakes.FAKE_SECTION1) + mock_get_configs.assert_called_once_with(fakes.FAKE_SECTION1) @ddt.ddt @@ -229,8 +182,7 @@ class TestTripleOYumRepoConfig(test_main.TestTripleoYumConfigBase): self.mock_object(os.path, 'isfile') self.mock_object(os, 'access') self.mock_object(os.path, 'isdir') - cfg_obj = yum_cfg.TripleOYumRepoConfig( - file_path=fakes.FAKE_FILE_PATH) + cfg_obj = yum_cfg.TripleOYumRepoConfig() mock_update = self.mock_object(yum_cfg.TripleOYumConfig, 'update_section') @@ -240,10 +192,12 @@ class TestTripleOYumRepoConfig(test_main.TestTripleoYumConfigBase): if enable is not None: expected_updates['enabled'] = '1' if enable else '0' - cfg_obj.update_section(fakes.FAKE_SECTION1, updates, enable=enable) + cfg_obj.update_section(fakes.FAKE_SECTION1, set_dict=updates, + file_path=fakes.FAKE_FILE_PATH, enabled=enable) mock_update.assert_called_once_with(fakes.FAKE_SECTION1, - expected_updates) + expected_updates, + file_path=fakes.FAKE_FILE_PATH) @ddt.ddt