Merge "Add module to generate CentOS Compose repos"

docs/yum_config.md

@@ -15,8 +15,8 @@ its repository and invoking in command line:
 
   This subcommand lets you enable or disable a repo and sets its configuration options.
   The *tripleo-yum-config* module will search for the provided repo name in all *.repo* files at REPO_DIR_PATH.
-  Optionally, you can provide a dir path where your repo files live or specify the full path of the repo file.
+  Optionally, you can provide a path where your repo files live or specify the full path of the repo file.
-  By default REPO_DIR_PATH is set to */etc/yum.repod.d/*.
+  By default REPO_DIR_PATH is set to */etc/yum.repos.d/*.
 
   Examples:
   ```
@@ -48,6 +48,19 @@ its repository and invoking in command line:
   ```
   sudo python -m tripleo_yum_config global --set-opts keepcache=1 cachedir="/var/cache/dnf"
   ```
+
+* **enable-compose-repos**
+
+  This subcommand will enable a list os yum repos based on the metadata retrieved from the `compose-url`.
+  The *tripleo-yum-config* module will create new repo files at REPO_DIR_PATH and enable them.
+  Optionally, you can provide a path where your repo files live, specify the variants that should be created and which repos need to be disabled afterwards.
+  By default REPO_DIR_PATH is set to */etc/yum.repos.d/*.
+
+  Example:
+  ```
+  sudo python -m tripleo_yum_config enable-compose-repos --compose-url https://composes.centos.org/latest-CentOS-Stream-8/compose/ --release centos-stream-8 --disable-all-conflicting
+  ```
+
 #### Install using setup.py
 
 Installation using python setup.py requires sudo, because the python source

molecule/default/converge.yml

@@ -2,8 +2,8 @@
 - name: Converge
   hosts: all
   tasks:
-
+    - name: "Test yum_config repo config"
-    - name: "Test yum_config repo"
+      become: true
       tripleo.repos.yum_config:
         type: repo
         name: appstream
@@ -14,12 +14,41 @@
         - molecule-idempotence-notest
       when: ansible_distribution_major_version is version(8, '>=')
 
+    - name: "Test yum_config global config"
+      become: true
+      tripleo.repos.yum_config:
+        type: global
+        file_path: /etc/dnf/dnf.conf
+        set_options:
+          skip_if_unavailable: "False"
+          keepcache: "0"
+      tags:
+        # TODO: fix yum_config to correctly report changed state and uncomment
+        # the line below which disables molecule idemptotence test.
+        - molecule-idempotence-notest
+
     - name: "Check get_hash"
       tripleo.repos.get_hash:
         release: master
+
     - name: "Check get_hash with invalid url"
       tripleo.repos.get_hash:
         release: master
         dlrn_url: 'https://httpbin.org/status/404'
       register: result
       failed_when: result is success
+
+    - name: "Test yum_config enable-compose-repos"
+      become: true
+      tripleo.repos.yum_config:
+        type: enable-compose-repos
+        compose_url: https://composes.centos.org/latest-CentOS-Stream-8/compose/
+        centos_release: centos-stream-8
+        variants:
+          - AppStream
+          - BaseOS
+        disable_repos:
+           - /etc/yum.repos.d/CentOS-Stream-AppStream.repo
+           - /etc/yum.repos.d/CentOS-Stream-BaseOS.repo
+      tags:
+        - molecule-idempotence-notest

plugins/module_utils/tripleo_repos/yum_config/__main__.py

@@ -17,8 +17,10 @@ import argparse
 import logging
 import sys
 
-import tripleo_repos.yum_config.yum_config as cfg
+import tripleo_repos.yum_config.compose_repos as compose_repos
+import tripleo_repos.yum_config.constants as const
 import tripleo_repos.yum_config.dnf_manager as dnf_mgr
+import tripleo_repos.yum_config.yum_config as cfg
 
 
 def options_to_dict(options):
@@ -63,8 +65,9 @@ def main():
     repo_args_parser.add_argument(
         '--config-dir-path',
         dest='config_dir_path',
+        default=const.YUM_REPO_DIR,
         help=(
-            'set the absolute directory path that holds all repo or module '
+            'set the absolute directory path that holds all repo '
             'configuration files')
     )
 
@@ -98,13 +101,61 @@ def main():
         help="sets module profile"
     )
 
+    # Compose repo arguments
+    compose_args_parser = argparse.ArgumentParser(add_help=False)
+    compose_args_parser.add_argument(
+        '--compose-url',
+        dest='compose_url',
+        required=True,
+        help='CentOS compose URL'
+    )
+    compose_args_parser.add_argument(
+        '--release',
+        dest='release',
+        choices=const.COMPOSE_REPOS_RELEASES,
+        default='centos-stream-8',
+        help='target CentOS release.'
+    )
+    compose_args_parser.add_argument(
+        '--arch',
+        choices=const.COMPOSE_REPOS_SUPPORTED_ARCHS,
+        default='x86_64',
+        help='set the architecture for the destination repos.'
+    )
+    compose_args_parser.add_argument(
+        '--disable-repos',
+        nargs='+',
+        help='list of repo names or repo absolute file paths to be disabled.'
+    )
+    compose_args_parser.add_argument(
+        '--disable-all-conflicting',
+        action='store_true',
+        dest='disable_conflicting',
+        default=False,
+        help='after enabling compose repos, disable all other repos that '
+             'match variant names.'
+    )
+    compose_args_parser.add_argument(
+        '--variants',
+        nargs='+',
+        help='Name of the repos to be enabled. Default behavior is to enable '
+             'all that match a specific release and architecture.'
+    )
+    compose_args_parser.add_argument(
+        '--config-dir-path',
+        dest='config_dir_path',
+        default=const.YUM_REPO_DIR,
+        help='set the absolute directory path that holds all repo '
+             'configuration files'
+    )
+
     # Common file path argument
     common_parse = argparse.ArgumentParser(add_help=False)
     common_parse.add_argument(
         '--config-file-path',
         dest='config_file_path',
         help=('set the absolute file path of the configuration file to be '
-              'updated')
+              'updated.')
     )
 
     # Main parser
@@ -133,6 +184,11 @@ def main():
         parents=[common_parse, options_parse],
         help='updates global yum configuration options'
     )
+    subparsers.add_parser(
+        'enable-compose-repos',
+        parents=[compose_args_parser],
+        help='enable CentOS compose repos based on an compose url.'
+    )
 
     args = main_parser.parse_args()
     if args.command is None:
@@ -146,10 +202,11 @@ def main():
     if args.command == 'repo':
         set_dict = options_to_dict(args.set_opts)
         config_obj = cfg.TripleOYumRepoConfig(
-            file_path=args.config_file_path,
             dir_path=args.config_dir_path)
 
-        config_obj.update_section(args.name, set_dict, enable=args.enable)
+        config_obj.update_section(args.name, set_dict,
+                                  file_path=args.config_file_path,
+                                  enabled=args.enable)
 
     elif args.command == 'module':
         dnf_mod_mgr = dnf_mgr.DnfModuleManager()
@@ -163,6 +220,19 @@ def main():
 
         config_obj.update_section('main', set_dict)
 
+    elif args.command == 'enable-compose-repos':
+        repo_obj = compose_repos.TripleOYumComposeRepoConfig(
+            args.compose_url,
+            args.release,
+            dir_path=args.config_dir_path,
+            arch=args.arch)
+
+        repo_obj.enable_compose_repos(variants=args.variants,
+                                      override_repos=args.disable_conflicting)
+        if args.disable_repos:
+            for file in args.disable_repos:
+                repo_obj.update_all_sections(file, enabled=False)
+
 
 def cli_entrypoint():
     try:

plugins/module_utils/tripleo_repos/yum_config/compose_repos.py

@@ -0,0 +1,202 @@
+#  Copyright 2021 Red Hat, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License. You may obtain
+#  a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations
+#  under the License.
+#
+from __future__ import (absolute_import, division, print_function)
+
+import logging
+import json
+import os
+import re
+import urllib.parse
+import urllib.request
+
+from .constants import (
+    YUM_REPO_DIR,
+    YUM_REPO_FILE_EXTENSION,
+    YUM_REPO_SUPPORTED_OPTIONS,
+    COMPOSE_REPOS_RELEASES,
+    COMPOSE_REPOS_INFO_PATH,
+    COMPOSE_REPOS_URL_PATTERN,
+    COMPOSE_REPOS_URL_REPLACE_STR,
+)
+from .exceptions import (
+    TripleOYumConfigInvalidSection,
+    TripleOYumConfigComposeError,
+)
+from .yum_config import (
+    TripleOYumConfig
+)
+
+__metaclass__ = type
+
+
+class TripleOYumComposeRepoConfig(TripleOYumConfig):
+    """Manages yum repo configuration files for CentOS Compose."""
+
+    def __init__(self, compose_url, release, dir_path=None, arch=None):
+        conf_dir_path = dir_path or YUM_REPO_DIR
+        self.arch = arch or 'x86_64'
+
+        # 1. validate release name
+        if release not in COMPOSE_REPOS_RELEASES:
+            msg = 'CentOS release not supported.'
+            raise TripleOYumConfigComposeError(error_msg=msg)
+        self.release = release
+
+        # 2. Validate URL
+        pattern = re.compile(COMPOSE_REPOS_URL_PATTERN[self.release])
+        if not pattern.match(compose_url):
+            msg = 'The provided URL does not match the expect pattern.'
+            raise TripleOYumConfigComposeError(error_msg=msg)
+
+        # 3. Get compose info from url
+        segments = [compose_url,
+                    COMPOSE_REPOS_INFO_PATH[self.release]]
+        self.compose_info_url = '/'.join(s.strip('/') for s in segments)
+        self.compose_info = self._get_compose_info()
+
+        # 4. Get compose-id from metadata
+        self.compose_id = self.compose_info['compose']['id']
+
+        # 5. Replace the compose-id from url to avoid 'labels'
+        repl_args = {'compose_id': self.compose_id}
+        self.compose_url = (
+            pattern.sub(
+                COMPOSE_REPOS_URL_REPLACE_STR[self.release] % repl_args,
+                compose_url)
+        )
+
+        super(TripleOYumComposeRepoConfig, self).__init__(
+            valid_options=YUM_REPO_SUPPORTED_OPTIONS,
+            dir_path=conf_dir_path,
+            file_extension=YUM_REPO_FILE_EXTENSION)
+
+    def _get_compose_info(self):
+        """Retrieve compose info for a provided compose-id url."""
+        # NOTE(dviroel): works for both centos 8 and 9
+        try:
+            logging.debug("Retrieving compose info from url: %s",
+                          self.compose_info_url)
+            res = urllib.request.urlopen(self.compose_info_url)
+        except Exception:
+            msg = ("Failed to retrieve compose info from url: %s"
+                   % self.compose_info_url)
+            raise TripleOYumConfigComposeError(error_msg=msg)
+        compose_info = json.loads(res.read())
+        if compose_info['header']['version'] != "1.2":
+            # NOTE(dviroel): Log a warning just in case we receive a different
+            #  version here. Code may fail depending on the change.
+            logging.warning("Expecting compose info version '1.2' but got %s.",
+                            compose_info['header']['version'])
+        return compose_info['payload']
+
+    def _get_repo_name(self, variant):
+        return " ".join([self.compose_id, variant])
+
+    def _get_repo_filename(self, variant):
+        return "-".join([self.compose_id, variant]) + '.repo'
+
+    def _get_repo_base_url(self, variant):
+        """Build the base_url based on variant name and system architecture."""
+        variant_info = self.compose_info['variants'][variant]
+        if not variant_info['paths'].get('repository', {}).get(self.arch):
+            # Variant has no support yet
+            return None
+        segments = [self.compose_url,
+                    variant_info['paths']['repository'][self.arch]]
+        return '/'.join(s.strip('/') for s in segments)
+
+    def get_compose_variants(self):
+        return self.compose_info['variants'].keys()
+
+    def enable_compose_repos(self, variants=None, override_repos=False):
+        """Enable CentOS compose repos of a given variant list.
+
+        This function will build from scratch all repos for a given compose-id
+        url. If a list of variants is not provided, it will enable all for all
+        variants returned from compose info.
+
+        :param variants: A list of variant names to be enabled.
+        :param override_repos: True if all matching variants in the same
+            repo directory should be disable in favor of the new repos.
+        """
+        if variants:
+            for var in variants:
+                if not (var in self.compose_info['variants'].keys()):
+                    msg = 'One or more provided variants are invalid.'
+                    raise TripleOYumConfigComposeError(error_msg=msg)
+
+        else:
+            variants = self.compose_info['variants'].keys()
+
+        updated_repos = {}
+        for var in variants:
+            base_url = self._get_repo_base_url(var)
+            if not base_url:
+                continue
+            add_dict = {
+                'name': self._get_repo_name(var),
+                'baseurl': base_url,
+                'enabled': '1',
+                'gpgcheck': '0',
+            }
+            filename = self._get_repo_filename(var)
+            file_path = os.path.join(self.dir_path, filename)
+            # create a file if doesn't exist and add a section to it
+            try:
+                self.add_section(var.lower(), add_dict, file_path)
+            except TripleOYumConfigInvalidSection:
+                logging.debug("Section '%s' that already exists in this file. "
+                              "Skipping...", var)
+            # needed to override other repos
+            updated_repos[var.lower()] = file_path
+
+        if override_repos:
+            for var in updated_repos:
+                config_files = self._get_config_files(var)
+                for file in config_files:
+                    if file != updated_repos[var]:
+                        msg = ("Disabling matching section '%(section)s' in "
+                               "configuration file: %(file)s.")
+                        msg_args = {
+                            'section': var,
+                            'file': file,
+                        }
+                        logging.debug(msg, msg_args)
+                        self.update_section(var, enabled=False, file_path=file)
+
+    def add_section(self, section, add_dict, file_path):
+        # Create a new file if it does not exists
+        if not os.path.isfile(file_path):
+            with open(file_path, '+w'):
+                pass
+        super(TripleOYumComposeRepoConfig, self).add_section(
+            section, add_dict, file_path)
+
+    def update_section(
+            self, section, set_dict=None, enabled=None, file_path=None):
+        update_dict = set_dict or {}
+        if enabled is not None:
+            update_dict['enabled'] = '1' if enabled else '0'
+        if update_dict:
+            super(TripleOYumComposeRepoConfig, self).update_section(
+                section, update_dict, file_path=file_path)
+
+    def update_all_sections(self, file_path, set_dict=None, enabled=None):
+        update_dict = set_dict or {}
+        if enabled is not None:
+            update_dict['enabled'] = '1' if enabled else '0'
+        if update_dict:
+            super(TripleOYumComposeRepoConfig, self).update_all_sections(
+                update_dict, file_path)

plugins/module_utils/tripleo_repos/yum_config/constants.py

@@ -39,3 +39,32 @@ YUM_REPO_FILE_EXTENSION = '.repo'
 Default constants for yum/dnf global configurations.
 """
 YUM_GLOBAL_CONFIG_FILE_PATH = '/etc/yum.conf'
+
+"""
+CentOS Stream compose repos defaults
+"""
+COMPOSE_REPOS_RELEASES = [
+    "centos-stream-8",
+    "centos-stream-9"
+]
+
+COMPOSE_REPOS_SUPPORTED_ARCHS = [
+    "aarch64",
+    "ppc64le",
+    "x86_64"
+]
+
+COMPOSE_REPOS_URL_PATTERN = {
+    "centos-stream-8": r"(^https:.*.centos.org/)([^/]*)(/compose/?$)",
+    "centos-stream-9": r"(^https:.*.centos.org/.*/)(.*)(/compose/?$)",
+}
+
+COMPOSE_REPOS_URL_REPLACE_STR = {
+    "centos-stream-8": r"\1%(compose_id)s\3",
+    "centos-stream-9": r"\1%(compose_id)s\3",
+}
+
+COMPOSE_REPOS_INFO_PATH = {
+    "centos-stream-8": "metadata/composeinfo.json",
+    "centos-stream-9": "metadata/composeinfo.json",
+}

plugins/module_utils/tripleo_repos/yum_config/exceptions.py

@@ -60,3 +60,10 @@ class TripleOYumConfigInvalidOption(Base):
 
     def __init__(self, error_msg):
         super(TripleOYumConfigInvalidOption, self).__init__(error_msg)
+
+
+class TripleOYumConfigComposeError(Base):
+    """An error occurred while configuring CentOS compose repos."""
+
+    def __init__(self, error_msg):
+        super(TripleOYumConfigComposeError, self).__init__(error_msg)

plugins/module_utils/tripleo_repos/yum_config/yum_config.py

@@ -31,13 +31,18 @@ from .exceptions import (
     TripleOYumConfigInvalidOption,
     TripleOYumConfigInvalidSection,
     TripleOYumConfigNotFound,
-    TripleOYumConfigPermissionDenied,
 )
 
 
 __metaclass__ = type
 
 
+def validated_file_path(file_path):
+    if os.path.isfile(file_path) and os.access(file_path, os.W_OK):
+        return True
+    return False
+
+
 class TripleOYumConfig:
     """
     This class is a base class for updating yum configuration files in
@@ -74,171 +79,230 @@ class TripleOYumConfig:
                 logger.addHandler(handler)
         logger.setLevel(logging.INFO)
 
-    def __init__(self, valid_options=None, file_path=None, dir_path=None,
+    def __init__(self, valid_options=None, dir_path=None, file_extension=None):
-                 file_extension=None):
         """
         Creates a TripleOYumConfig object that holds configuration file
         information.
 
         :param valid_options: A list of options that can be updated on this
             file.
-        :param file_path: The file path to configuration file to be updated.
         :param dir_path: The directory path that this class can use to search
             for configuration files to be updated.
         :param: file_extension: File extension to filter configuration files
             in the search directory.
         """
-        self.config_file_path = file_path
         self.dir_path = dir_path
         self.file_extension = file_extension
         self.valid_options = valid_options
 
         # Sanity checks
-        if not (file_path or dir_path):
-            msg = ('A configuration file path or a directory path must be '
-                   'provided.')
-            raise TripleOYumConfigNotFound(error_msg=msg)
-
-        if file_path:
-            if not os.path.isfile(file_path):
-                msg = ('The configuration file "{0}" was not found in the '
-                       'provided path.').format(file_path)
-                raise TripleOYumConfigNotFound(error_msg=msg)
-            if not os.access(file_path, os.W_OK):
-                msg = ('The configuration file {0} is not '
-                       'writable.'.format(file_path))
-                raise TripleOYumConfigPermissionDenied(error_msg=msg)
-
         if dir_path:
             if not os.path.isdir(dir_path):
                 msg = ('The configuration dir "{0}" was not found in the '
                        'provided path.').format(dir_path)
                 raise TripleOYumConfigNotFound(error_msg=msg)
 
-    def _read_config_file(self, section):
+    def _read_config_file(self, file_path, section=None):
-        """Read the configuration file associate with this object.
+        """Reads a configuration file.
 
-        If no configuration file is provided, this method will search for
+        :param section: The name of the section that will be update. Only used
-        'section' name in all files inside the configuration directory. The
+            to fail earlier if the section is not found.
-        first occurrence of 'section' will be returned, and a warning will be
+        :return: a config parser object and the full file path.
-        logged if more than one configuration file has the same 'section' set.
-
-        :param section: The name of the section to be updated.
-        :return: a config parser object and the file path.
         """
         config = configparser.ConfigParser()
-        # A) A configuration file path was provided.
+        file_paths = [file_path]
-        if self.config_file_path:
+        if self.dir_path:
-            try:
+            # if dir_path is configured, we can search for filename there
-                config.read(self.config_file_path)
+            file_paths.append(os.path.join(self.dir_path, file_path))
-            except configparser.Error:
-                msg = 'Unable to parse configuration file {0}.'.format(
-                    self.config_file_path)
-                raise TripleOYumConfigFileParseError(error_msg=msg)
 
-            if section not in config.sections():
+        valid_file_path = None
-                msg = ('The provided section "{0}" was not found in the '
+        for file in file_paths:
-                       'configuration file {1}.').format(
+            if validated_file_path(file):
-                    section, self.config_file_path)
+                valid_file_path = file
-                raise TripleOYumConfigInvalidSection(error_msg=msg)
+                break
+        if not valid_file_path:
+            msg = ('The configuration file "{0}" was '
+                   'not found.'.format(file_path))
+            raise TripleOYumConfigNotFound(error_msg=msg)
 
-            return config, self.config_file_path
+        try:
+            config.read(valid_file_path)
+        except configparser.Error:
+            msg = 'Unable to parse configuration file {0}.'.format(
+                valid_file_path)
+            raise TripleOYumConfigFileParseError(error_msg=msg)
 
-        # B) Search for a configuration file that has the provided section
+        if section and section not in config.sections():
-        section_found = False
+            msg = ('The provided section "{0}" was not found in the '
-        config_file_path = None
+                   'configuration file {1}.').format(
-        for file in os.listdir(self.dir_path):
+                section, valid_file_path)
-            # Skip files that don't match the file extension or are not
+            raise TripleOYumConfigInvalidSection(error_msg=msg)
-            # writable
-            if self.file_extension and not file.endswith(
-                    self.file_extension):
-                continue
-            if not os.access(os.path.join(self.dir_path, file), os.W_OK):
-                continue
 
-            tmp_config = configparser.ConfigParser()
+        return config, valid_file_path
-            try:
-                tmp_config.read(os.path.join(self.dir_path, file))
-            except configparser.Error:
-                continue
-            if section in tmp_config.sections():
-                if section_found:
-                    logging.warning('Section "%s" is listed more than once in '
-                                    'configuration files.', section)
-                else:
-                    # Read the first occurrence of 'section'
-                    config_file_path = os.path.join(self.dir_path, file)
-                    config.read(config_file_path)
-                    section_found = True
 
-        return config, config_file_path
+    def _get_config_files(self, section):
+        """Gets all configuration file paths for a given section.
 
-    def update_section(self, section, set_dict):
+        This method will search for a 'section' name in all files inside the
-        """Updates a set of options for a specified section.
+        configuration directory. All files with 'section' will be returned.
+
+        :param section: Section to be found inside configuration files.
+        :return: A list of config file paths.
+        """
+        # Search for a configuration file that has the provided section
+        config_files_path = []
+        if section and self.dir_path:
+            for file in os.listdir(self.dir_path):
+                # Skip files that don't match the file extension or are not
+                # writable
+                if self.file_extension and not file.endswith(
+                        self.file_extension):
+                    continue
+                if not os.access(os.path.join(self.dir_path, file), os.W_OK):
+                    continue
+
+                tmp_config = configparser.ConfigParser()
+                try:
+                    tmp_config.read(os.path.join(self.dir_path, file))
+                except configparser.Error:
+                    continue
+                if section in tmp_config.sections():
+                    config_files_path.append(os.path.join(self.dir_path, file))
+
+        return config_files_path
+
+    def update_section(self, section, set_dict, file_path=None):
+        """Updates a set of options of a section.
+
+        If a file path is not provided by the caller, this function will search
+        for the section in all files located in the working directory and
+        update each one of them.
 
         :param section: Name of the section on the configuration file that will
             be updated.
         :param set_dict: Dict with all options and values to be updated in the
             configuration file section.
+        :param file_path: Path to the configuration file to be updated.
         """
         if self.valid_options:
             if not all(key in self.valid_options for key in set_dict.keys()):
                 msg = 'One or more provided options are not valid.'
                 raise TripleOYumConfigInvalidOption(error_msg=msg)
 
-        config, config_file_path = self._read_config_file(section)
+        files = [file_path] if file_path else self._get_config_files(section)
-        if not (config and config_file_path):
+        if not files:
-            msg = ('The provided section "{0}" was not found within any '
+            msg = ('No configuration files were found for the provided '
-                   'configuration file.').format(section)
+                   'section {0}'.format(section))
             raise TripleOYumConfigNotFound(error_msg=msg)
 
-        # Update configuration file with dict updates
+        for file in files:
-        config[section].update(set_dict)
+            config, file = self._read_config_file(file, section=section)
+            # Update configuration file with dict updates
+            config[section].update(set_dict)
+            with open(file, 'w') as f:
+                config.write(f)
 
-        with open(config_file_path, 'w') as file:
+        logging.info("Section '%s' was successfully "
+                     "updated.", section)
+
+    def add_section(self, section, add_dict, file_path):
+        """ Adds a new section with options in a provided config file.
+
+        :param section: Section name to be added to the config file.
+        :param add_dict: Dict with all options and values to be added into the
+            new section.
+        :param file_path: Path to the configuration file to be updated.
+        """
+        # This section shouldn't exist in the provided file
+        config, file_path = self._read_config_file(file_path=file_path)
+        if section in config.sections():
+            msg = ("Section '%s' already exists in the configuration "
+                   "file.", section)
+            raise TripleOYumConfigInvalidSection(error_msg=msg)
+
+        # Add new section
+        config.add_section(section)
+        # Update configuration file with dict updates
+        config[section].update(add_dict)
+
+        with open(file_path, '+w') as file:
             config.write(file)
 
-        logging.info("Section '%s' was successfully updated.", section)
+        logging.info("Section '%s' was successfully "
+                     "added.", section)
+
+    def update_all_sections(self, set_dict, file_path):
+        """Updates all section of a given configuration file.
+
+        :param set_dict: Dict with all options and values to be updated in
+            the configuration file.
+        :param file_path: Path to the configuration file to be updated.
+        """
+        if self.valid_options:
+            if not all(key in self.valid_options for key in set_dict.keys()):
+                msg = 'One or more provided options are not valid.'
+                raise TripleOYumConfigInvalidOption(error_msg=msg)
+
+        config, file_path = self._read_config_file(file_path)
+        for section in config.sections():
+            config[section].update(set_dict)
+
+        with open(file_path, '+w') as file:
+            config.write(file)
+
+        logging.info("All sections for '%s' were successfully "
+                     "updated.", file_path)
 
 
 class TripleOYumRepoConfig(TripleOYumConfig):
     """Manages yum repo configuration files."""
 
-    def __init__(self, file_path=None, dir_path=None):
+    def __init__(self, dir_path=None):
-        if file_path:
-            logging.info(
-                "Using '%s' as yum repo configuration file.", file_path)
         conf_dir_path = dir_path or YUM_REPO_DIR
 
         super(TripleOYumRepoConfig, self).__init__(
             valid_options=YUM_REPO_SUPPORTED_OPTIONS,
-            file_path=file_path,
             dir_path=conf_dir_path,
             file_extension=YUM_REPO_FILE_EXTENSION)
 
-    def update_section(self, section, set_dict, enable=None):
+    def update_section(
-        if enable is not None:
+            self, section, set_dict=None, file_path=None, enabled=None):
-            set_dict['enabled'] = '1' if enable else '0'
+        update_dict = set_dict or {}
-
+        if enabled is not None:
-        super(TripleOYumRepoConfig, self).update_section(section, set_dict)
+            update_dict['enabled'] = '1' if enabled else '0'
+        if update_dict:
+            super(TripleOYumRepoConfig, self).update_section(
+                section, update_dict, file_path=file_path)
 
 
 class TripleOYumGlobalConfig(TripleOYumConfig):
     """Manages yum global configuration file."""
 
     def __init__(self, file_path=None):
-        conf_file_path = file_path or YUM_GLOBAL_CONFIG_FILE_PATH
+        self.conf_file_path = file_path or YUM_GLOBAL_CONFIG_FILE_PATH
         logging.info("Using '%s' as yum global configuration "
-                     "file.", conf_file_path)
+                     "file.", self.conf_file_path)
-        if file_path is None:
+        if file_path is not None:
+            # validate user provided file path
+            validated_file_path(file_path)
+        else:
             # If there is no default 'yum.conf' configuration file, we need to
             # create it. If the user specify another conf file that doesn't
             # exists, the operation will fail.
-            if not os.path.isfile(conf_file_path):
+            if not os.path.isfile(self.conf_file_path):
                 config = configparser.ConfigParser()
-                config.read(conf_file_path)
+                config.read(self.conf_file_path)
                 config.add_section('main')
-                with open(conf_file_path, '+w') as file:
+                with open(self.conf_file_path, '+w') as file:
                     config.write(file)
 
-        super(TripleOYumGlobalConfig, self).__init__(file_path=conf_file_path)
+        super(TripleOYumGlobalConfig, self).__init__()
+
+    def update_section(self, section, set_dict, file_path=None):
+        super(TripleOYumGlobalConfig, self).update_section(
+            section, set_dict, file_path=(file_path or self.conf_file_path))
+
+    def add_section(self, section, set_dict, file_path=None):
+        add_file_path = file_path or self.conf_file_path
+        super(TripleOYumGlobalConfig, self).add_section(
+            section, set_dict, add_file_path)

plugins/modules/yum_config.py

@@ -24,7 +24,7 @@ options:
           - The type of yum configuration to be changed.
         required: true
         type: str
-        choices: [repo, module, global]
+        choices: [repo, module, global, 'enable-compose-repos']
     name:
         description:
           - Name of the repo or module to be changed. This options is
@@ -65,6 +65,40 @@ options:
           - Absolute path of the directory that contains the configuration
             file to be changed.
         type: path
+        default: /etc/yum.repos.d
+    compose_url:
+        description:
+          - URL that contains CentOS compose repositories.
+        type: str
+    centos_release:
+        description:
+          - Target CentOS release.
+        type: str
+        choices: [centos-stream-8, centos-stream-9]
+    arch:
+        description:
+          - System architecture which the repos will be configure.
+        type: str
+        choices: [aarch64, ppc64le, x86_64]
+        default: x86_64
+    variants:
+        description:
+          - Repository variants that should be configured. If not provided,
+            all available variants will be configured.
+        type: list
+        elements: str
+    disable_conflicting_variants:
+        description:
+          - Disable all repos from the same directory that match variants'
+            name.
+        type: bool
+        default: false
+    disable_repos:
+        description:
+          - List with file path of repos that should be disabled after
+            successfully enabling all compose repos.
+        type: list
+        elements: str
 
 author:
     - Douglas Viroel (@viroel)
@@ -113,6 +147,20 @@ EXAMPLES = r'''
     set_options:
       skip_if_unavailable: "False"
       keepcache: "0"
+
+- name: Configure a set of repos based on latest CentOS Stream 8 compose
+  become: true
+  become_user: root
+  tripleo_yup_config:
+    compose_url: https://composes.centos.org/latest-CentOS-Stream-8/compose/
+    centos_release: centos-stream-8
+    variants:
+      - AppStream
+      - BaseOS
+    disable_conflicting_variants: true
+    disable_repos:
+      - /etc/yum.repos.d/CentOS-Linux-AppStream.repo
+      - /etc/yum.repos.d/CentOS-Linux-BaseOS.repo
 '''
 
 RETURN = r''' # '''
@@ -121,8 +169,14 @@ from ansible.module_utils.basic import AnsibleModule  # noqa: E402
 
 
 def run_module():
+    try:
+        import ansible_collections.tripleo.repos.plugins.module_utils. \
+            tripleo_repos.yum_config.constants as const
+    except ImportError:
+        import tripleo_repos.yum_config.constants as const
     # define available arguments/parameters a user can pass to the module
-    supported_config_types = ['repo', 'module', 'global']
+    supported_config_types = ['repo', 'module', 'global',
+                              'enable-compose-repos']
     supported_module_operations = ['install', 'remove', 'reset']
     module_args = dict(
         type=dict(type='str', required=True, choices=supported_config_types),
@@ -133,7 +187,17 @@ def run_module():
         profile=dict(type='str'),
         set_options=dict(type='dict', default={}),
         file_path=dict(type='path'),
-        dir_path=dict(type='path'),
+        dir_path=dict(type='path', default=const.YUM_REPO_DIR),
+        compose_url=dict(type='str'),
+        centos_release=dict(type='str',
+                            choices=const.COMPOSE_REPOS_RELEASES),
+        arch=dict(type='str', choices=const.COMPOSE_REPOS_SUPPORTED_ARCHS,
+                  default='x86_64'),
+        variants=dict(type='list', default=[],
+                      elements='str'),
+        disable_conflicting_variants=dict(type='bool', default=False),
+        disable_repos=dict(type='list', default=[],
+                           elements='str'),
     )
 
     module = AnsibleModule(
@@ -141,6 +205,7 @@ def run_module():
         required_if=[
             ["type", "repo", ["name"]],
             ["type", "module", ["name"]],
+            ["type", "enable-compose-repos", ["compose_url"]],
         ],
         supports_check_mode=False
     )
@@ -162,18 +227,36 @@ def run_module():
                 tripleo_repos.yum_config.dnf_manager as dnf_mgr
             import ansible_collections.tripleo.repos.plugins.module_utils.\
                 tripleo_repos.yum_config.yum_config as cfg
+            import ansible_collections.tripleo.repos.plugins.module_utils. \
+                tripleo_repos.yum_config.compose_repos as repos
         except ImportError:
             import tripleo_repos.yum_config.dnf_manager as dnf_mgr
             import tripleo_repos.yum_config.yum_config as cfg
+            import tripleo_repos.yum_config.compose_repos as repos
 
         if module.params['type'] == 'repo':
             config_obj = cfg.TripleOYumRepoConfig(
-                file_path=module.params['file_path'],
                 dir_path=module.params['dir_path'])
             config_obj.update_section(
                 module.params['name'],
                 m_set_opts,
-                enable=module.params['enabled'])
+                file_path=module.params['file_path'],
+                enabled=module.params['enabled'])
+
+        elif module.params['type'] == 'enable-compose-repos':
+            # 1. Create compose repo config object
+            repo_obj = repos.TripleOYumComposeRepoConfig(
+                module.params['compose_url'],
+                module.params['centos_release'],
+                dir_path=module.params['dir_path'],
+                arch=module.params['arch'])
+            # 2. enable CentOS compose repos
+            repo_obj.enable_compose_repos(
+                variants=module.params['variants'],
+                override_repos=module.params['disable_conflicting_variants'])
+            # 3. Disable all repos provided in disable_repos
+            for file in module.params['disable_repos']:
+                repo_obj.update_all_sections(file, enabled=False)
 
         elif module.params['type'] == 'module':
             dnf_mod_mgr = dnf_mgr.DnfModuleManager()

tests/sanity/ignore.txt

@@ -1,2 +1,3 @@
 plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py replace-urlopen
 plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py pylint:ansible-bad-import
+plugins/module_utils/tripleo_repos/yum_config/compose_repos.py replace-urlopen

tests/unit/yum_config/test_main.py

@@ -20,8 +20,9 @@ from unittest import mock
 from . import fakes
 from . import mock_modules  # noqa: F401
 import tripleo_repos.yum_config.__main__ as main
-import tripleo_repos.yum_config.yum_config as yum_cfg
+import tripleo_repos.yum_config.constants as const
 import tripleo_repos.yum_config.dnf_manager as dnf_mgr
+import tripleo_repos.yum_config.yum_config as yum_cfg
 
 
 class TestTripleoYumConfigBase(unittest.TestCase):
@@ -56,10 +57,10 @@ class TestTripleoYumConfigMain(TestTripleoYumConfigBase):
         main.main()
         expected_dict = {'key1': 'value1', 'key2': 'value2'}
 
-        mock_yum_repo_obj.assert_called_once_with(
+        mock_yum_repo_obj.assert_called_once_with(dir_path=const.YUM_REPO_DIR)
-            file_path=fakes.FAKE_FILE_PATH, dir_path=None)
         mock_update_section.assert_called_once_with(
-            'fake_repo', expected_dict, enable=True)
+            'fake_repo', expected_dict, file_path=fakes.FAKE_FILE_PATH,
+            enabled=True)
 
     @ddt.data('enable', 'disable', 'reset', 'install', 'remove')
     def test_main_module(self, operation):

tests/unit/yum_config/test_yum_config.py

@@ -29,51 +29,25 @@ import tripleo_repos.yum_config.yum_config as yum_cfg
 class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase):
     """Tests for TripleYumConfig class and its methods."""
 
-    def _create_yum_config_obj(self, file_path=None, dir_path=None,
+    def _create_yum_config_obj(self, dir_path=None, valid_options=None,
-                               valid_options=None, file_extension=None):
+                               file_extension=None):
         self.mock_object(os.path, 'isfile')
         self.mock_object(os, 'access')
         self.mock_object(os.path, 'isdir')
-        return yum_cfg.TripleOYumConfig(file_path=file_path, dir_path=dir_path,
+        return yum_cfg.TripleOYumConfig(dir_path=dir_path,
                                         valid_options=valid_options,
                                         file_extension=file_extension)
 
-    @ddt.data(
+    def test_tripleo_yum_config_invalid_dir_path(self):
-        {'file_path': None, 'dir_path': None, 'is_file_ret': None,
-         'access_ret': None, 'is_dir_ret': None,
-         'exception': exc.TripleOYumConfigNotFound},
-
-        {'file_path': 'fake_path', 'dir_path': None, 'is_file_ret': False,
-         'access_ret': None, 'is_dir_ret': None,
-         'exception': exc.TripleOYumConfigNotFound},
-
-        {'file_path': 'fake_path', 'dir_path': None, 'is_file_ret': True,
-         'access_ret': False, 'is_dir_ret': None,
-         'exception': exc.TripleOYumConfigPermissionDenied},
-
-        {'file_path': None, 'dir_path': 'fake_dir', 'is_file_ret': None,
-         'access_ret': None, 'is_dir_ret': False,
-         'exception': exc.TripleOYumConfigNotFound},
-    )
-    @ddt.unpack
-    def test_tripleo_yum_config_invalid_parameters(
-            self, file_path, dir_path, is_file_ret, access_ret, is_dir_ret,
-            exception):
-        self.mock_object(os.path, 'isfile',
-                         mock.Mock(return_value=is_file_ret))
-        self.mock_object(os, 'access',
-                         mock.Mock(return_value=access_ret))
         self.mock_object(os.path, 'isdir',
-                         mock.Mock(return_value=is_dir_ret))
+                         mock.Mock(return_value=False))
 
-        self.assertRaises(exception,
+        self.assertRaises(exc.TripleOYumConfigNotFound,
                           yum_cfg.TripleOYumConfig,
-                          file_path=file_path,
+                          dir_path='fake_dir_path')
-                          dir_path=dir_path)
 
     def test_read_config_file_path(self):
-        yum_config = self._create_yum_config_obj(
+        yum_config = self._create_yum_config_obj()
-            file_path=fakes.FAKE_FILE_PATH)
 
         parser_mock = mock.Mock()
         self.mock_object(configparser, 'ConfigParser',
@@ -83,7 +57,8 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase):
                          mock.Mock(return_value=fakes.FAKE_SECTIONS))
 
         config_parser, file_path = yum_config._read_config_file(
-            fakes.FAKE_SECTION1
+            fakes.FAKE_FILE_PATH,
+            section=fakes.FAKE_SECTION1
         )
 
         self.assertEqual(parser_mock, config_parser)
@@ -91,8 +66,7 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase):
         read_mock.assert_called_once_with(fakes.FAKE_FILE_PATH)
 
     def test_read_config_file_path_parse_error(self):
-        yum_config = self._create_yum_config_obj(
+        yum_config = self._create_yum_config_obj()
-            file_path=fakes.FAKE_FILE_PATH)
 
         parser_mock = mock.Mock()
         self.mock_object(configparser, 'ConfigParser',
@@ -102,13 +76,13 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase):
 
         self.assertRaises(exc.TripleOYumConfigFileParseError,
                           yum_config._read_config_file,
-                          fakes.FAKE_SECTION1)
+                          fakes.FAKE_FILE_PATH,
+                          section=fakes.FAKE_SECTION1)
 
         read_mock.assert_called_once_with(fakes.FAKE_FILE_PATH)
 
     def test_read_config_file_path_invalid_section(self):
-        yum_config = self._create_yum_config_obj(
+        yum_config = self._create_yum_config_obj()
-            file_path=fakes.FAKE_FILE_PATH)
 
         parser_mock = mock.Mock()
         self.mock_object(configparser, 'ConfigParser',
@@ -119,11 +93,12 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase):
 
         self.assertRaises(exc.TripleOYumConfigInvalidSection,
                           yum_config._read_config_file,
-                          'invalid_section')
+                          fakes.FAKE_FILE_PATH,
+                          section='invalid_section')
 
         read_mock.assert_called_once_with(fakes.FAKE_FILE_PATH)
 
-    def test_read_config_file_dir(self):
+    def test_get_config_files(self):
         yum_config = self._create_yum_config_obj(
             dir_path=fakes.FAKE_DIR_PATH,
             file_extension='.conf')
@@ -133,50 +108,28 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase):
             parser_mocks.append(parser_mock)
             self.mock_object(parser_mock, 'read')
 
-        self.mock_object(parser_mocks[1], 'sections',
+        self.mock_object(parser_mocks[0], 'sections',
                          mock.Mock(return_value=[]))
         # second file inside dir will have the expected sections
-        self.mock_object(parser_mocks[2], 'sections',
+        self.mock_object(parser_mocks[1], 'sections',
                          mock.Mock(return_value=fakes.FAKE_SECTIONS))
+        self.mock_object(parser_mocks[2], 'sections',
+                         mock.Mock(return_value=[]))
         self.mock_object(os, 'listdir',
                          mock.Mock(return_value=fakes.FAKE_DIR_FILES))
         self.mock_object(os, 'access', mock.Mock(return_value=True))
         self.mock_object(configparser, 'ConfigParser',
                          mock.Mock(side_effect=parser_mocks))
 
-        config_parser, file_path = yum_config._read_config_file(
+        result = yum_config._get_config_files(fakes.FAKE_SECTION1)
-            fakes.FAKE_SECTION1)
+        expected_dir_path = [os.path.join(fakes.FAKE_DIR_PATH,
-        expected_dir_path = os.path.join(fakes.FAKE_DIR_PATH,
+                                          fakes.FAKE_DIR_FILES[1])]
-                                         fakes.FAKE_DIR_FILES[1])
 
-        self.assertEqual(parser_mocks[0], config_parser)
+        self.assertEqual(expected_dir_path, result)
-        self.assertEqual(expected_dir_path, file_path)
-
-    def test_read_config_file_dir_section_not_found(self):
-        yum_config = self._create_yum_config_obj(
-            dir_path=fakes.FAKE_DIR_PATH,
-            file_extension='.conf')
-        parser_mock = mock.Mock()
-        self.mock_object(parser_mock, 'read')
-        self.mock_object(parser_mock, 'sections',
-                         mock.Mock(return_value=[]))
-        self.mock_object(configparser, 'ConfigParser',
-                         mock.Mock(return_value=parser_mock))
-
-        self.mock_object(os, 'listdir',
-                         mock.Mock(return_value=fakes.FAKE_DIR_FILES))
-        self.mock_object(os, 'access', mock.Mock(return_value=True))
-
-        config_parser, file_path = yum_config._read_config_file(
-            fakes.FAKE_SECTION1)
-
-        self.assertEqual(parser_mock, config_parser)
-        self.assertIsNone(file_path)
 
     @mock.patch('builtins.open')
     def test_update_section(self, open):
         yum_config = self._create_yum_config_obj(
-            file_path=fakes.FAKE_FILE_PATH,
             valid_options=fakes.FAKE_SUPP_OPTIONS)
         config_parser = fakes.FakeConfigParser({fakes.FAKE_SECTION1: {}})
 
@@ -186,13 +139,14 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase):
 
         updates = {fakes.FAKE_OPTION1: 'new_fake_value'}
 
-        yum_config.update_section(fakes.FAKE_SECTION1, updates)
+        yum_config.update_section(fakes.FAKE_SECTION1, updates,
+                                  file_path=fakes.FAKE_FILE_PATH)
 
-        mock_read_config.assert_called_once_with(fakes.FAKE_SECTION1)
+        mock_read_config.assert_called_once_with(fakes.FAKE_FILE_PATH,
+                                                 section=fakes.FAKE_SECTION1)
 
     def test_update_section_invalid_options(self):
         yum_config = self._create_yum_config_obj(
-            file_path=fakes.FAKE_FILE_PATH,
             valid_options=fakes.FAKE_SUPP_OPTIONS)
 
         updates = {'invalid_option': 'new_fake_value'}
@@ -200,15 +154,15 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase):
         self.assertRaises(exc.TripleOYumConfigInvalidOption,
                           yum_config.update_section,
                           fakes.FAKE_SECTION1,
-                          updates)
+                          updates,
+                          file_path=fakes.FAKE_FILE_PATH)
 
     def test_update_section_file_not_found(self):
         yum_config = self._create_yum_config_obj(
-            file_path=fakes.FAKE_FILE_PATH,
             valid_options=fakes.FAKE_SUPP_OPTIONS)
-        mock_read_config = self.mock_object(
+        mock_get_configs = self.mock_object(
-            yum_config, '_read_config_file',
+            yum_config, '_get_config_files',
-            mock.Mock(return_value=(mock.Mock(), None)))
+            mock.Mock(return_value=[]))
 
         updates = {fakes.FAKE_OPTION1: 'new_fake_value'}
 
@@ -216,8 +170,7 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase):
                           yum_config.update_section,
                           fakes.FAKE_SECTION1,
                           updates)
-
+        mock_get_configs.assert_called_once_with(fakes.FAKE_SECTION1)
-        mock_read_config.assert_called_once_with(fakes.FAKE_SECTION1)
 
 
 @ddt.ddt
@@ -229,8 +182,7 @@ class TestTripleOYumRepoConfig(test_main.TestTripleoYumConfigBase):
         self.mock_object(os.path, 'isfile')
         self.mock_object(os, 'access')
         self.mock_object(os.path, 'isdir')
-        cfg_obj = yum_cfg.TripleOYumRepoConfig(
+        cfg_obj = yum_cfg.TripleOYumRepoConfig()
-            file_path=fakes.FAKE_FILE_PATH)
 
         mock_update = self.mock_object(yum_cfg.TripleOYumConfig,
                                        'update_section')
@@ -240,10 +192,12 @@ class TestTripleOYumRepoConfig(test_main.TestTripleoYumConfigBase):
         if enable is not None:
             expected_updates['enabled'] = '1' if enable else '0'
 
-        cfg_obj.update_section(fakes.FAKE_SECTION1, updates, enable=enable)
+        cfg_obj.update_section(fakes.FAKE_SECTION1, set_dict=updates,
+                               file_path=fakes.FAKE_FILE_PATH, enabled=enable)
 
         mock_update.assert_called_once_with(fakes.FAKE_SECTION1,
-                                            expected_updates)
+                                            expected_updates,
+                                            file_path=fakes.FAKE_FILE_PATH)
 
 
 @ddt.ddt