Browse Source

yum-config: add support to download repo from url

yum-config now support downloading a repo file from a url and
make updates on its options.
A new parameter 'down-url' was added to the cli and ansible module.

Change-Id: I6dba307be5e66892ecfba04df7766ae8dbbfa71e
Signed-off-by: Douglas Viroel <dviroel@redhat.com>
changes/11/816211/7
Douglas Viroel 7 months ago
parent
commit
50c7cf01fd
  1. 13
      docs/yum_config.md
  2. 13
      molecule/default/converge.yml
  3. 8
      molecule/default/verify.yml
  4. 2
      plugins/module_utils/tripleo_repos/utils.py
  5. 27
      plugins/module_utils/tripleo_repos/yum_config/__main__.py
  6. 3
      plugins/module_utils/tripleo_repos/yum_config/constants.py
  7. 7
      plugins/module_utils/tripleo_repos/yum_config/exceptions.py
  8. 97
      plugins/module_utils/tripleo_repos/yum_config/yum_config.py
  9. 42
      plugins/modules/yum_config.py
  10. 1
      tests/unit/yum_config/fakes.py
  11. 44
      tests/unit/yum_config/test_main.py
  12. 101
      tests/unit/yum_config/test_yum_config.py

13
docs/yum_config.md

@ -20,9 +20,18 @@ its repository and invoking in command line:
Examples:
```
sudo python -m tripleo_yum_config repo appstream --enable --set-opts baseurl=http://newbaseurl exclude="package*"
sudo python -m tripleo_yum_config repo epel --disable --config-dir-path=/path/to/yum.repos.d
sudo python -m tripleo_yum_config repo --name appstream --enable --set-opts baseurl=http://newbaseurl exclude="package*"
sudo python -m tripleo_yum_config repo --name epel --disable --config-dir-path=/path/to/yum.repos.d
```
The parameter *--down-url* can be used to retrieve a configuration file from a URL and populate the destination
configuration file with all its content. When used together with *--name*, only the requested repo name will be
updated in the process.
Examples:
```
sudo python -m tripleo_yum_config repo --down-url http://remoterepofile.repo --enable --set-opts priority=20 --config-file-path=/path/to/file.repo
sudo python -m tripleo_yum_config repo --name appstream --down-url http://remoterepofile.repo --enable
```
* **module**
This subcommand lets you enable, disable, remove, install or reset a module.

13
molecule/default/converge.yml

@ -71,3 +71,16 @@
- molecule-idempotence-notest
# NOTE: operation available only for CentOS >= 8
when: ansible_distribution_major_version is version(8, '>=')
- name: "Test create repo from repo file"
become: true
tripleo.repos.yum_config:
type: repo
enabled: true
file_path: "/etc/yum.repos.d/delorean.repo"
down_url: "https://trunk.rdoproject.org/centos8-master/current-tripleo/delorean.repo"
set_options:
priority: "20"
tags:
# TODO: fix yum_config to correctly report changed state and uncomment
# the line below which disables molecule idempotence test.
- molecule-idempotence-notest

8
molecule/default/verify.yml

@ -37,3 +37,11 @@
- "BaseOS"
# NOTE: operation available only for CentOS >= 8
when: ansible_distribution_major_version is version(8, '>=')
- name: Check if 'priority' was set to 20 in 'delorean-component-compute'
include_tasks: assert_ini_key_value.yml
with_items:
- name: "delorean-component-compute"
path: "/etc/yum.repos.d/delorean.repo"
section: "delorean-component-compute"
key: priority
value: "20"

2
plugins/module_utils/tripleo_repos/utils.py

@ -40,7 +40,7 @@ else:
def http_get(url):
try:
response = open_url(url, method='GET')
return (response.read(), response.status)
return (response.read().decode('utf-8'), response.status)
except Exception as e:
return (str(e), -1)
except ImportError:

27
plugins/module_utils/tripleo_repos/yum_config/__main__.py

@ -49,7 +49,7 @@ def main():
# Repo arguments
repo_args_parser = argparse.ArgumentParser(add_help=False)
repo_args_parser.add_argument(
'name',
'--name',
help='name of the repo to be modified'
)
@ -85,6 +85,13 @@ def main():
'set the absolute directory path that holds all repo '
'configuration files')
)
repo_args_parser.add_argument(
'--down-url',
dest='down_url',
help=(
'URL of a repo file to be used as base to create or update '
'a repo configuration file.')
)
# Generic key-value options
options_parse = argparse.ArgumentParser(add_help=False)
@ -227,9 +234,21 @@ def main():
config_obj = cfg.TripleOYumRepoConfig(
dir_path=args.config_dir_path,
environment_file=args.env_file)
config_obj.add_or_update_section(args.name, set_dict=set_dict,
file_path=args.config_file_path,
enabled=args.enable)
if args.name is not None:
config_obj.add_or_update_section(args.name, set_dict=set_dict,
file_path=args.config_file_path,
enabled=args.enable,
from_url=args.down_url)
else:
# When no section (name) is provided, we consider all sections from
# repo file downloaded from the URL, otherwise fail.
if args.down_url is None:
logging.error("You must provide a repo 'name' or a valid "
"'url' where repo info can be downloaded.")
sys.exit(2)
config_obj.add_or_update_all_sections_from_url(
args.down_url, file_path=args.config_file_path,
set_dict=set_dict, enabled=args.enable)
elif args.command == 'module':
import tripleo_repos.yum_config.dnf_manager as dnf_mgr

3
plugins/module_utils/tripleo_repos/yum_config/constants.py

@ -33,7 +33,8 @@ YUM_REPO_SUPPORTED_OPTIONS = [
'mirrorlist',
'module_hotfixes',
'name',
'priority'
'priority',
'skip_if_unavailable',
]
"""

7
plugins/module_utils/tripleo_repos/yum_config/exceptions.py

@ -67,3 +67,10 @@ class TripleOYumConfigComposeError(Base):
def __init__(self, error_msg):
super(TripleOYumConfigComposeError, self).__init__(error_msg)
class TripleOYumConfigUrlError(Base):
"""An error occurred while fetching repo from the url."""
def __init__(self, error_msg):
super(TripleOYumConfigUrlError, self).__init__(error_msg)

97
plugins/module_utils/tripleo_repos/yum_config/yum_config.py

@ -15,6 +15,7 @@
from __future__ import (absolute_import, division, print_function)
import io
import logging
import os
import subprocess
@ -31,7 +32,14 @@ from .exceptions import (
TripleOYumConfigInvalidOption,
TripleOYumConfigInvalidSection,
TripleOYumConfigNotFound,
TripleOYumConfigUrlError,
)
try:
import tripleo_repos.utils as repos_utils
except ImportError:
import ansible_collections.tripleo.repos.plugins.module_utils.\
tripleo_repos.utils as repos_utils
py_version = sys.version_info.major
if py_version < 3:
@ -296,6 +304,30 @@ class TripleOYumConfig:
logging.info("All sections for '%s' were successfully "
"updated.", file_path)
def get_config_from_url(self, url):
content, status = repos_utils.http_get(url)
if status != 200:
msg = ("Invalid response code received from provided url: "
"{0}. Response code: {1}."
).format(url, status)
logging.error(msg)
raise TripleOYumConfigUrlError(error_msg=msg)
config = cfg_parser.ConfigParser()
if py_version < 3:
sfile = io.StringIO(content)
config.readfp(sfile)
else:
config.read_string(content)
return config
def get_options_from_url(self, url, section):
config = self.get_config_from_url(url)
if section not in config.sections():
msg = ("Section '{0}' was not found in the configuration file "
"provided by the url {1}.").format(section, url)
raise TripleOYumConfigInvalidSection(error_msg=msg)
return dict(config.items(section))
class TripleOYumRepoConfig(TripleOYumConfig):
"""Manages yum repo configuration files."""
@ -310,26 +342,42 @@ class TripleOYumRepoConfig(TripleOYumConfig):
environment_file=environment_file)
def update_section(
self, section, set_dict=None, file_path=None, enabled=None):
update_dict = set_dict or {}
self, section, set_dict=None, file_path=None, enabled=None,
from_url=None):
update_dict = (
self.get_options_from_url(from_url, section) if from_url else {})
if set_dict:
update_dict.update(set_dict)
if enabled is not None:
update_dict['enabled'] = '1' if enabled else '0'
if update_dict:
super(TripleOYumRepoConfig, self).update_section(
section, update_dict, file_path=file_path)
def add_section(self, section, add_dict, file_path, enabled=None):
update_dict = add_dict or {}
def add_section(self, section, add_dict, file_path, enabled=None,
from_url=None):
update_dict = (
self.get_options_from_url(from_url, section) if from_url else {})
update_dict.update(add_dict)
if enabled is not None:
update_dict['enabled'] = '1' if enabled else '0'
super(TripleOYumRepoConfig, self).add_section(
section, update_dict, file_path)
def add_or_update_section(self, section, set_dict=None, file_path=None,
enabled=None, create_if_not_exists=True):
def add_or_update_section(self, section, set_dict=None,
file_path=None, enabled=None,
create_if_not_exists=True, from_url=None):
new_set_dict = (
self.get_options_from_url(from_url, section) if from_url else {})
new_set_dict.update(set_dict)
# make sure that it has a name
if 'name' not in new_set_dict.keys():
new_set_dict['name'] = section
# Try to update existing repos
try:
self.update_section(
section, set_dict=set_dict, file_path=file_path,
section, set_dict=new_set_dict, file_path=file_path,
enabled=enabled)
except TripleOYumConfigNotFound:
if not create_if_not_exists or file_path is None:
@ -338,10 +386,33 @@ class TripleOYumRepoConfig(TripleOYumConfig):
# Create a new file if it does not exists
with open(file_path, 'w+'):
pass
# When creating a new repo file, make sure that it has a name
if 'name' not in set_dict.keys():
set_dict['name'] = section
self.add_section(section, set_dict, file_path, enabled=enabled)
self.add_section(section, new_set_dict, file_path, enabled=enabled)
except TripleOYumConfigInvalidSection:
self.add_section(section, new_set_dict, file_path, enabled=enabled)
def add_or_update_all_sections_from_url(
self, from_url, file_path=None, set_dict=None, enabled=None,
create_if_not_exists=True):
"""Adds or updates all sections based on repo file from a URL."""
tmp_config = self.get_config_from_url(from_url)
if file_path is None:
# Build a file_path based on download url. If not compatible,
# don't fill file_path and let the code search for sections in all
# repo files inside config dir_path.
file_name = from_url.split('/')[-1]
if file_name.endswith(".repo"):
# Expecting a '*.repo' filename here, since the file can't be
# created with a different extension
file_path = os.path.join(self.dir_path, file_name)
for section in tmp_config.sections():
update_dict = dict(tmp_config.items(section))
update_dict.update(set_dict)
self.add_or_update_section(
section, set_dict=update_dict,
file_path=file_path, enabled=enabled,
create_if_not_exists=create_if_not_exists)
class TripleOYumGlobalConfig(TripleOYumConfig):
@ -372,7 +443,7 @@ class TripleOYumGlobalConfig(TripleOYumConfig):
super(TripleOYumGlobalConfig, self).update_section(
section, set_dict, file_path=(file_path or self.conf_file_path))
def add_section(self, section, set_dict, file_path=None):
def add_section(self, section, add_dict, file_path=None):
add_file_path = file_path or self.conf_file_path
super(TripleOYumGlobalConfig, self).add_section(
section, set_dict, add_file_path)
section, add_dict, add_file_path)

42
plugins/modules/yum_config.py

@ -28,7 +28,8 @@ options:
name:
description:
- Name of the repo or module to be changed. This options is
mandatory only for repo and module types.
mandatory only for 'repo' when no 'down_url' is provided. This
options is always mandatory for 'module' type.
type: str
enabled:
description:
@ -36,6 +37,13 @@ options:
- This options is ignored for yum global configuration.
type: bool
default: true
down_url:
description:
- URL of a downloadable repo file to be used as base to construct a
new repo file. When used together with 'name', will update only the
requested section, without a specific section 'name' will add or
update all sections available in the downloaded file.
type: str
operation:
description:
- Operation to be execute within a dnf module.
@ -178,11 +186,11 @@ def run_module():
try:
import ansible_collections.tripleo.repos.plugins.module_utils. \
tripleo_repos.yum_config.constants as const
import ansible_collections.tripleo.repos.plugins.module_utils. \
tripleo_repos.yum_config.utils as utils
from ansible_collections.tripleo.repos.plugins.module_utils. \
tripleo_repos.yum_config import utils
except ImportError:
import tripleo_repos.yum_config.constants as const
import tripleo_repos.yum_config.utils as utils
from tripleo_repos.yum_config import utils
supported_config_types = ['repo', 'global', 'module',
'enable-compose-repos']
@ -191,6 +199,7 @@ def run_module():
type=dict(type='str', required=True, choices=supported_config_types),
name=dict(type='str'),
enabled=dict(type='bool', default=True),
down_url=dict(type='str'),
operation=dict(type='str', choices=supported_module_operations),
stream=dict(type='str'),
profile=dict(type='str'),
@ -210,7 +219,6 @@ def run_module():
elements='str'),
)
required_if_params = [
["type", "repo", ["name"]],
["type", "module", ["name"]],
["type", "enable-compose-repos", ["compose_url"]]
]
@ -227,6 +235,12 @@ def run_module():
"supported with python 2.").format(module.params['type'])
module.fail_json(msg=msg)
if (module.params['type'] == 'repo' and not
module.params['name'] and not module.params['down_url']):
msg = ("When using configuration type '{0}' you must provide a repo "
"'name' or a 'down_url'.").format(module.params['type'])
module.fail_json(msg=msg)
distro, major_version, __ = utils.get_distro_info()
dnf_module_support = False
for min_distro_ver in const.DNF_MODULE_MINIMAL_DISTRO_VERSIONS:
@ -262,11 +276,19 @@ def run_module():
config_obj = cfg.TripleOYumRepoConfig(
dir_path=module.params['dir_path'],
environment_file=module.params['environment_file'])
config_obj.add_or_update_section(
module.params['name'],
set_dict=m_set_opts,
file_path=module.params['file_path'],
enabled=module.params['enabled'])
if module.params['name']:
config_obj.add_or_update_section(
module.params['name'],
set_dict=m_set_opts,
file_path=module.params['file_path'],
enabled=module.params['enabled'],
from_url=module.params['down_url'])
else:
config_obj.add_or_update_all_sections_from_url(
module.params['down_url'],
set_dict=m_set_opts,
file_path=module.params['file_path'],
enabled=module.params['enabled'])
elif module.params['type'] == 'global':
config_obj = cfg.TripleOYumGlobalConfig(

1
tests/unit/yum_config/fakes.py

@ -24,6 +24,7 @@ FAKE_SET_DICT = {
'key1': 'value1',
'key2': 'value2',
}
FAKE_REPO_DOWN_URL = '/fake/down/url/fake.repo'
FAKE_COMPOSE_URL = (
'https://composes.centos.org/fake-CentOS-Stream/compose/')

44
tests/unit/yum_config/test_main.py

@ -51,9 +51,10 @@ class TestTripleoYumConfigMain(TestTripleoYumConfigBase):
mock.Mock(return_value=("centos", "8", None)))
def test_main_repo(self):
sys.argv[1:] = ['repo', 'fake_repo', '--enable',
sys.argv[1:] = ['repo', '--name', 'fake_repo', '--enable',
'--set-opts', 'key1=value1', 'key2=value2',
'--config-file-path', fakes.FAKE_FILE_PATH]
'--config-file-path', fakes.FAKE_FILE_PATH,
'--down-url', fakes.FAKE_REPO_DOWN_URL]
yum_repo_obj = mock.Mock()
mock_update_section = self.mock_object(yum_repo_obj,
@ -69,7 +70,30 @@ class TestTripleoYumConfigMain(TestTripleoYumConfigBase):
environment_file=None)
mock_update_section.assert_called_once_with(
'fake_repo', set_dict=expected_dict,
file_path=fakes.FAKE_FILE_PATH, enabled=True)
file_path=fakes.FAKE_FILE_PATH, enabled=True,
from_url=fakes.FAKE_REPO_DOWN_URL)
def test_main_repo_from_url(self):
sys.argv[1:] = ['repo', '--enable',
'--set-opts', 'key1=value1', 'key2=value2',
'--config-file-path', fakes.FAKE_FILE_PATH,
'--down-url', fakes.FAKE_REPO_DOWN_URL]
yum_repo_obj = mock.Mock()
mock_update_all_sections = self.mock_object(
yum_repo_obj, 'add_or_update_all_sections_from_url')
mock_yum_repo_obj = self.mock_object(
yum_cfg, 'TripleOYumRepoConfig',
mock.Mock(return_value=yum_repo_obj))
main.main()
expected_dict = {'key1': 'value1', 'key2': 'value2'}
mock_yum_repo_obj.assert_called_once_with(dir_path=const.YUM_REPO_DIR,
environment_file=None)
mock_update_all_sections.assert_called_once_with(
fakes.FAKE_REPO_DOWN_URL, file_path=fakes.FAKE_FILE_PATH,
set_dict=expected_dict, enabled=True)
@ddt.data('enable', 'disable', 'reset', 'install', 'remove')
def test_main_module(self, operation):
@ -113,7 +137,19 @@ class TestTripleoYumConfigMain(TestTripleoYumConfigBase):
@ddt.data('repo')
def test_main_repo_mod_without_name(self, command):
sys.argv[1:] = [command, '--set-opts', 'key1=value1']
sys.argv[1:] = [command, '--set-opts', 'key1=value1',
'--config-dir-path', '/tmp']
with self.assertRaises(SystemExit) as command:
main.main()
self.assertEqual(2, command.exception.code)
def test_main_repo_without_name_and_url(self):
sys.argv[1:] = ['repo', '--enable',
'--set-opts', 'key1=value1', 'key2=value2',
'--config-file-path', fakes.FAKE_FILE_PATH,
'--config-dir-path', '/tmp']
with self.assertRaises(SystemExit) as command:
main.main()

101
tests/unit/yum_config/test_yum_config.py

@ -24,6 +24,7 @@ from . import test_main
import tripleo_repos.yum_config.constants as const
import tripleo_repos.yum_config.exceptions as exc
import tripleo_repos.yum_config.yum_config as yum_cfg
import tripleo_repos.utils as repos_utils
@ddt.ddt
@ -249,6 +250,48 @@ class TestTripleOYumConfig(test_main.TestTripleoYumConfigBase):
shell=True)
env_update_mock.assert_called_once_with(exp_env_dict)
def test_get_config_from_url_invalid_url(self):
yum_config = self._create_yum_config_obj(
valid_options=fakes.FAKE_SUPP_OPTIONS)
fake_context = mock.Mock()
self.mock_object(repos_utils, 'http_get',
mock.Mock(return_value=(fake_context, 404)))
self.assertRaises(exc.TripleOYumConfigUrlError,
yum_config.get_config_from_url,
fakes.FAKE_REPO_DOWN_URL)
def test_get_config_from_url(self):
yum_config = self._create_yum_config_obj(
valid_options=fakes.FAKE_SUPP_OPTIONS)
fake_context = mock.Mock()
self.mock_object(repos_utils, 'http_get',
mock.Mock(return_value=(fake_context, 200)))
parser_mock = mock.Mock()
self.mock_object(configparser, 'ConfigParser',
mock.Mock(return_value=parser_mock))
result = yum_config.get_config_from_url(fakes.FAKE_REPO_DOWN_URL)
self.assertEqual(parser_mock, result)
def test_get_options_from_url_section_not_found(self):
yum_config = self._create_yum_config_obj(
valid_options=fakes.FAKE_SUPP_OPTIONS)
fake_config = mock.Mock()
self.mock_object(fake_config, 'sections',
mock.Mock(return_value=[]))
mock_get_from_url = self.mock_object(
yum_config, 'get_config_from_url',
mock.Mock(return_value=fake_config))
self.assertRaises(exc.TripleOYumConfigInvalidSection,
yum_config.get_options_from_url,
fakes.FAKE_REPO_DOWN_URL,
fakes.FAKE_SECTION1)
mock_get_from_url.assert_called_once_with(fakes.FAKE_REPO_DOWN_URL)
@ddt.ddt
class TestTripleOYumRepoConfig(test_main.TestTripleoYumConfigBase):
@ -283,26 +326,37 @@ class TestTripleOYumRepoConfig(test_main.TestTripleoYumConfigBase):
file_path=fakes.FAKE_FILE_PATH)
@mock.patch('builtins.open')
def test_add_or_update_section(self, open):
@ddt.data(None, fakes.FAKE_REPO_DOWN_URL)
def test_add_or_update_section(self, open, down_url):
mock_update = self.mock_object(
self.config_obj, 'update_section',
mock.Mock(side_effect=exc.TripleOYumConfigNotFound(
error_msg='error')))
mock_add_section = self.mock_object(self.config_obj, 'add_section')
extra_opt = {'key1': 'new value 1'}
mock_get_from_url = self.mock_object(
self.config_obj, 'get_options_from_url',
mock.Mock(return_value=extra_opt))
self.config_obj.add_or_update_section(
fakes.FAKE_SECTION1,
set_dict=fakes.FAKE_SET_DICT,
file_path=fakes.FAKE_FILE_PATH,
enabled=True,
create_if_not_exists=True)
create_if_not_exists=True,
from_url=down_url)
fake_set_dict = copy.deepcopy(fakes.FAKE_SET_DICT)
fake_set_dict['name'] = fakes.FAKE_SECTION1
if down_url:
fake_set_dict.update(extra_opt)
mock_get_from_url.assert_called_once_with(down_url,
fakes.FAKE_SECTION1)
mock_update.assert_called_once_with(fakes.FAKE_SECTION1,
set_dict=fakes.FAKE_SET_DICT,
set_dict=fake_set_dict,
file_path=fakes.FAKE_FILE_PATH,
enabled=True)
fake_set_dict = copy.deepcopy(fakes.FAKE_SET_DICT)
fake_set_dict['name'] = fakes.FAKE_SECTION1
mock_add_section.assert_called_once_with(
fakes.FAKE_SECTION1,
fake_set_dict,
@ -327,8 +381,10 @@ class TestTripleOYumRepoConfig(test_main.TestTripleoYumConfigBase):
enabled=True,
create_if_not_exists=create_if_not_exists)
fake_set_dict = copy.deepcopy(fakes.FAKE_SET_DICT)
fake_set_dict['name'] = fakes.FAKE_SECTION1
mock_update.assert_called_once_with(fakes.FAKE_SECTION1,
set_dict=fakes.FAKE_SET_DICT,
set_dict=fake_set_dict,
file_path=fake_path,
enabled=True)
@ -346,6 +402,39 @@ class TestTripleOYumRepoConfig(test_main.TestTripleoYumConfigBase):
mock_add.assert_called_once_with(fakes.FAKE_SECTION1, updated_dict,
fakes.FAKE_FILE_PATH)
@ddt.data(fakes.FAKE_FILE_PATH, None)
def test_add_or_update_all_sections_from_url(self, file_path):
add_or_update_section = self.mock_object(
self.config_obj, 'add_or_update_section')
fake_config = mock.Mock()
self.mock_object(fake_config, 'sections',
mock.Mock(return_value=[fakes.FAKE_SECTION1]))
options_from_url = {'key3': 'value3'}
self.mock_object(fake_config, 'items',
mock.Mock(return_value=options_from_url))
mock_get_from_url = self.mock_object(
self.config_obj, 'get_config_from_url',
mock.Mock(return_value=fake_config))
exp_file_path = (
file_path or os.path.join(
'/tmp', fakes.FAKE_REPO_DOWN_URL.split('/')[-1])
)
self.config_obj.add_or_update_all_sections_from_url(
fakes.FAKE_REPO_DOWN_URL,
file_path=file_path,
set_dict=fakes.FAKE_SET_DICT,
enabled=True,
create_if_not_exists=True)
mock_get_from_url.assert_called_once_with(fakes.FAKE_REPO_DOWN_URL)
expected_update_dict = copy.deepcopy(fakes.FAKE_SET_DICT)
expected_update_dict.update(options_from_url)
add_or_update_section.assert_called_once_with(
fakes.FAKE_SECTION1, set_dict=expected_update_dict,
file_path=exp_file_path, enabled=True,
create_if_not_exists=True)
@ddt.ddt
class TestTripleOYumGlobalConfig(test_main.TestTripleoYumConfigBase):

Loading…
Cancel
Save