diff --git a/galaxy.yml b/galaxy.yml index 525c1f6..0f3f456 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -1,6 +1,6 @@ name: repos namespace: tripleo -version: 0.0.2 +version: 0.0.5 readme: README.md authors: - Red Hat diff --git a/plugins/module_utils/tripleo_repos/get_hash/constants.py b/plugins/module_utils/tripleo_repos/get_hash/constants.py index db24be3..e19e77e 100644 --- a/plugins/module_utils/tripleo_repos/get_hash/constants.py +++ b/plugins/module_utils/tripleo_repos/get_hash/constants.py @@ -36,3 +36,50 @@ This is the path that we expect to find the system installed config.yaml. The path is specified in [options.data_files] of the project setup.cfg. """ CONFIG_PATH = '/usr/local/etc/tripleo_get_hash/config.yaml' + +DEFAULT_CONFIG = { + "tripleo_releases": [ + "master", + "wallaby", + "victoria", + "ussuri", + "train", + "stein", + "queens", + "osp16-2", + "osp17" + ], + "dlrn_url": "https://trunk.rdoproject.org", + "rdo_named_tags": [ + "current", + "consistent", + "component-ci-testing", + "promoted-components", + "tripleo-ci-testing", + "current-tripleo", + "current-tripleo-rdo" + ], + "tripleo_ci_components": [ + "baremetal", + "cinder", + "clients", + "cloudops", + "common", + "compute", + "glance", + "manila", + "network", + "octavia", + "security", + "swift", + "tempest", + "tripleo", + "ui", + "validation" + ], + "os_versions": [ + "centos7", + "centos8", + "rhel8" + ] +} diff --git a/plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py b/plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py index 1e3c49d..4eba5ae 100644 --- a/plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py +++ b/plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py @@ -19,10 +19,28 @@ import logging import sys import os import yaml -import requests -from .constants import CONFIG_PATH, CONFIG_KEYS -from .exceptions import TripleOHashMissingConfig, TripleOHashInvalidConfig +from .constants import CONFIG_PATH, CONFIG_KEYS, DEFAULT_CONFIG +from .exceptions import ( + TripleOHashInvalidConfig) +from typing import Tuple +# portable http_get that uses either ansible recommended way or python native +# urllib. +try: + from ansible.module_utils.urls import open_url + def http_get(url: str) -> Tuple[str, int]: + response = open_url(url, method='GET') + return (response.read(), response.status) +except ImportError: + import gzip + from urllib.request import urlopen + + def http_get(url: str) -> Tuple[str, int]: + # https://stackoverflow.com/questions/35122232/urllib-request-urlopen-return-bytes-but-i-cannot-decode-it + response = urlopen(url) + return ( + gzip.decompress(response.read()).decode('utf-8'), + int(response.status)) __metaclass__ = type @@ -120,11 +138,8 @@ class TripleOHashInfo: elif local_config: config_path = local_config else: - raise TripleOHashMissingConfig( - "Configuration file not found at {0} or {1}".format( - CONFIG_PATH, local_config - ) - ) + logging.info("Using embedded config file") + return DEFAULT_CONFIG logging.info("Using config file at %s", config_path) with open(config_path, 'r') as config_yaml: loaded_config = yaml.safe_load(config_yaml) @@ -164,7 +179,7 @@ class TripleOHashInfo: repo_url = self._resolve_repo_url(config['dlrn_url']) self.dlrn_url = repo_url - repo_url_response = requests.get(repo_url).text + repo_url_response, status = http_get(repo_url) if repo_url.endswith('commit.yaml'): from_commit_yaml = self._hashes_from_commit_yaml(repo_url_response) self.full_hash = from_commit_yaml[0] diff --git a/plugins/modules/get_hash.py b/plugins/modules/get_hash.py index e0a75ab..7b44b16 100644 --- a/plugins/modules/get_hash.py +++ b/plugins/modules/get_hash.py @@ -112,12 +112,8 @@ def run_module(): try: - try: - from ansible_collections.tripleo.repos.plugins.module_utils.\ - tripleo_repo.get_hash.tripleo_hash_info import TripleOHashInfo - except ImportError: - from tripleo_repos.get_hash.tripleo_hash_info import \ - TripleOHashInfo + from ansible_collections.tripleo.repos.plugins.module_utils.\ + tripleo_repos.get_hash.tripleo_hash_info import TripleOHashInfo os_version = module.params.get('os_version') release = module.params.get('release') diff --git a/test-requirements.txt b/test-requirements.txt index 7cda47e..0f172a3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -13,4 +13,3 @@ testscenarios>=0.4 # Apache-2.0/BSD testtools>=1.4.0 # MIT stestr>=2.0.0 # Apache-2.0 fixtures>=3.0.0 # Apache-2.0/BSD -requests_mock>=1.8.0 # Apache-2.0 diff --git a/tests/sanity/requirements.txt b/tests/sanity/requirements.txt index 6c9fdba..5500f00 100644 --- a/tests/sanity/requirements.txt +++ b/tests/sanity/requirements.txt @@ -1,2 +1 @@ PyYAML -requests diff --git a/tests/unit/get_hash/test_tripleo_get_hash.py b/tests/unit/get_hash/test_tripleo_get_hash.py index f0a82f2..4104a92 100644 --- a/tests/unit/get_hash/test_tripleo_get_hash.py +++ b/tests/unit/get_hash/test_tripleo_get_hash.py @@ -14,11 +14,10 @@ # # -import requests_mock import sys import unittest from unittest import mock -from unittest.mock import mock_open +from unittest.mock import mock_open, MagicMock, patch import yaml import tripleo_repos.get_hash.exceptions as exc @@ -37,11 +36,10 @@ class TestGetHash(unittest.TestCase): """ def test_centos_8_current_tripleo_stable(self, mock_config): - with requests_mock.Mocker() as req_mock: - req_mock.get( - 'https://trunk.rdoproject.org/centos8-victoria/current-tripleo/delorean.repo.md5', # noqa - text=test_fakes.TEST_REPO_MD5, - ) + mocked = MagicMock( + return_value=(test_fakes.TEST_REPO_MD5, 200)) + with patch( + 'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked): args = ['--os-version', 'centos8', '--release', 'victoria'] sys.argv[1:] = args main_res = tgh.main() @@ -57,11 +55,10 @@ class TestGetHash(unittest.TestCase): args = ['--verbose'] debug_msgs = [] - with requests_mock.Mocker() as req_mock: - req_mock.get( - 'https://trunk.rdoproject.org/centos8-master/current-tripleo/delorean.repo.md5', # noqa - text=test_fakes.TEST_REPO_MD5, - ) + mocked = MagicMock( + return_value=(test_fakes.TEST_REPO_MD5, 200)) + with patch( + 'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked): with self.assertLogs() as captured: sys.argv[1:] = args tgh.main() @@ -74,11 +71,12 @@ class TestGetHash(unittest.TestCase): def test_verbose_logging_off(self, mock_config): debug_msgs = [] - with requests_mock.Mocker() as req_mock: - req_mock.get( - 'https://trunk.rdoproject.org/centos8-master/current-tripleo/delorean.repo.md5', # noqa - text=test_fakes.TEST_REPO_MD5, - ) + + mocked = MagicMock( + return_value=(test_fakes.TEST_REPO_MD5, 200)) + with patch( + 'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked): + args = ['--tag', 'current-tripleo', '--os-version', 'centos8'] with self.assertLogs() as captured: sys.argv[1:] = args @@ -101,14 +99,13 @@ class TestGetHash(unittest.TestCase): config_file.close() # interate for each of config components for component in config_yaml['tripleo_ci_components']: - with requests_mock.Mocker() as req_mock: - req_mock.get( - "https://trunk.rdoproject.org/centos8-master/component" - "/{}/current-tripleo/commit.yaml".format( - component - ), - text=test_fakes.TEST_COMMIT_YAML_COMPONENT, - ) + + mocked = MagicMock( + return_value=(test_fakes.TEST_COMMIT_YAML_COMPONENT, 200)) + with patch( + 'tripleo_repos.get_hash.tripleo_hash_info.http_get', + mocked): + args = ['--component', "{}".format(component)] sys.argv[1:] = args main_res = tgh.main() @@ -142,14 +139,12 @@ class TestGetHash(unittest.TestCase): config_file.close() # iterate for each of config named tags for tag in config_yaml['rdo_named_tags']: - with requests_mock.Mocker() as req_mock: - req_mock.get( - "https://trunk.rdoproject.org/centos8-master" - "/{}/delorean.repo.md5".format( - tag - ), - text=test_fakes.TEST_REPO_MD5, - ) + mocked = MagicMock( + return_value=(test_fakes.TEST_REPO_MD5, 200)) + with patch( + 'tripleo_repos.get_hash.tripleo_hash_info.http_get', + mocked): + args = ['--tag', "{}".format(tag)] sys.argv[1:] = args main_res = tgh.main() @@ -163,12 +158,12 @@ class TestGetHash(unittest.TestCase): self.assertEqual(tag, main_res.tag) def test_override_dlrn_url(self, mock_config): - with requests_mock.Mocker() as req_mock: - req_mock.get( - "https://awoo.com/awoo/centos8-master/current-tripleo" - "/delorean.repo.md5", - text=test_fakes.TEST_REPO_MD5, - ) + mocked = MagicMock( + return_value=(test_fakes.TEST_REPO_MD5, 200)) + with patch( + 'tripleo_repos.get_hash.tripleo_hash_info.http_get', + mocked): + args = ['--dlrn-url', 'https://awoo.com/awoo'] sys.argv[1:] = args main_res = tgh.main() @@ -179,11 +174,11 @@ class TestGetHash(unittest.TestCase): ) def test_override_os_version_release_rhel8(self, mock_config): - with requests_mock.Mocker() as req_mock: - req_mock.get( - "https://awoo.com/awoo/rhel8-osp16-2/current-tripleo" - "/delorean.repo.md5", text=test_fakes.TEST_REPO_MD5, - ) + mocked = MagicMock( + return_value=(test_fakes.TEST_REPO_MD5, 200)) + with patch( + 'tripleo_repos.get_hash.tripleo_hash_info.http_get', + mocked): args = [ '--dlrn-url', 'https://awoo.com/awoo', diff --git a/tests/unit/get_hash/test_tripleo_get_hash_info.py b/tests/unit/get_hash/test_tripleo_get_hash_info.py index 6e4a81e..ff5936d 100644 --- a/tests/unit/get_hash/test_tripleo_get_hash_info.py +++ b/tests/unit/get_hash/test_tripleo_get_hash_info.py @@ -18,9 +18,8 @@ import unittest import tripleo_repos.get_hash.tripleo_hash_info as thi import tripleo_repos.get_hash.exceptions as exc from . import fakes as test_fakes -import requests_mock from unittest import mock -from unittest.mock import mock_open +from unittest.mock import mock_open, MagicMock, patch @mock.patch( @@ -41,11 +40,10 @@ class TestGetHashInfo(unittest.TestCase): '1f5a41f31db8e3eb51caa9c0e201ab0583747be8', 'None', ) - with requests_mock.Mocker() as req_mock: - req_mock.get( - 'https://trunk.rdoproject.org/centos8-master/component/common/current-tripleo/commit.yaml', # noqa - text=test_fakes.TEST_COMMIT_YAML_COMPONENT, - ) + mocked = MagicMock( + return_value=(test_fakes.TEST_COMMIT_YAML_COMPONENT, 200)) + with patch( + 'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked): mock_hash_info = thi.TripleOHashInfo( 'centos8', 'master', 'common', 'current-tripleo' ) @@ -55,12 +53,10 @@ class TestGetHashInfo(unittest.TestCase): self.assertEqual(expected_result, actual_result) def test_resolve_repo_url_component_commit_yaml(self, mock_config): - with requests_mock.Mocker() as req_mock: - # test component url - req_mock.get( - 'https://trunk.rdoproject.org/centos8-master/component/common/current-tripleo/commit.yaml', # noqa - text=test_fakes.TEST_COMMIT_YAML_COMPONENT, - ) + mocked = MagicMock( + return_value=(test_fakes.TEST_COMMIT_YAML_COMPONENT, 200)) + with patch( + 'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked): c8_component_hash_info = thi.TripleOHashInfo( 'centos8', 'master', 'common', 'current-tripleo' ) @@ -71,13 +67,10 @@ class TestGetHashInfo(unittest.TestCase): ) def test_resolve_repo_url_centos8_repo_md5(self, mock_config): - with requests_mock.Mocker() as req_mock: - # test vanilla centos8 url - req_mock.get( - 'https://trunk.rdoproject.org/centos8-master/current-tripleo/delorean.repo.md5', # noqa - - text=test_fakes.TEST_REPO_MD5, - ) + mocked = MagicMock( + return_value=(test_fakes.TEST_REPO_MD5, 200)) + with patch( + 'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked): c8_hash_info = thi.TripleOHashInfo( 'centos8', 'master', None, 'current-tripleo' ) @@ -88,13 +81,10 @@ class TestGetHashInfo(unittest.TestCase): ) def test_resolve_repo_url_centos7_commit_yaml(self, mock_config): - with requests_mock.Mocker() as req_mock: - # test centos7 url - req_mock.get( - 'https://trunk.rdoproject.org/centos7-master/current-tripleo/commit.yaml', # noqa - - text=test_fakes.TEST_COMMIT_YAML_CENTOS_7, - ) + mocked = MagicMock( + return_value=(test_fakes.TEST_COMMIT_YAML_CENTOS_7, 200)) + with patch( + 'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked): c7_hash_info = thi.TripleOHashInfo( 'centos7', 'master', None, 'current-tripleo' ) @@ -105,12 +95,10 @@ class TestGetHashInfo(unittest.TestCase): ) def test_get_tripleo_hash_info_centos8_md5(self, mock_config): - with requests_mock.Mocker() as req_mock: - req_mock.get( - 'https://trunk.rdoproject.org/centos8-master/current-tripleo/delorean.repo.md5', # noqa - - text=test_fakes.TEST_REPO_MD5, - ) + mocked = MagicMock( + return_value=(test_fakes.TEST_REPO_MD5, 200)) + with patch( + 'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked): created_hash_info = thi.TripleOHashInfo( 'centos8', 'master', None, 'current-tripleo' ) @@ -126,11 +114,10 @@ class TestGetHashInfo(unittest.TestCase): expected_commit_hash = '476a52df13202a44336c8b01419f8b73b93d93eb' expected_distro_hash = '1f5a41f31db8e3eb51caa9c0e201ab0583747be8' expected_full_hash = '476a52df13202a44336c8b01419f8b73b93d93eb_1f5a41f3' # noqa - with requests_mock.Mocker() as req_mock: - req_mock.get( - 'https://trunk.rdoproject.org/centos8-victoria/component/common/tripleo-ci-testing/commit.yaml', # noqa - text=test_fakes.TEST_COMMIT_YAML_COMPONENT, - ) + mocked = MagicMock( + return_value=(test_fakes.TEST_COMMIT_YAML_COMPONENT, 200)) + with patch( + 'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked): created_hash_info = thi.TripleOHashInfo( 'centos8', 'victoria', 'common', 'tripleo-ci-testing' ) @@ -150,11 +137,10 @@ class TestGetHashInfo(unittest.TestCase): expected_commit_hash = 'b5ef03c9c939db551b03e9490edc6981ff582035' expected_distro_hash = '76ebc4655502820b7677579349fd500eeca292e6' expected_full_hash = 'b5ef03c9c939db551b03e9490edc6981ff582035_76ebc465' # noqa - with requests_mock.Mocker() as req_mock: - req_mock.get( - 'https://trunk.rdoproject.org/centos7-master/tripleo-ci-testing/commit.yaml', # noqa - text=test_fakes.TEST_COMMIT_YAML_CENTOS_7, - ) + mocked = MagicMock( + return_value=(test_fakes.TEST_COMMIT_YAML_CENTOS_7, 200)) + with patch( + 'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked): created_hash_info = thi.TripleOHashInfo( 'centos7', 'master', None, 'tripleo-ci-testing' ) @@ -169,11 +155,10 @@ class TestGetHashInfo(unittest.TestCase): self.assertEqual(created_hash_info.os_version, 'centos7') def test_bad_config_file(self, mock_config): - with requests_mock.Mocker() as req_mock: - req_mock.get( - 'https://trunk.rdoproject.org/centos7-master/tripleo-ci-testing/commit.yaml', # noqa - text=test_fakes.TEST_COMMIT_YAML_CENTOS_7, - ) + mocked = MagicMock( + return_value=test_fakes.TEST_COMMIT_YAML_CENTOS_7) + with patch( + 'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked): with mock.patch( 'builtins.open', new_callable=mock_open, @@ -188,30 +173,12 @@ class TestGetHashInfo(unittest.TestCase): 'tripleo-ci-testing', ) - def test_missing_config_file(self, mock_config): - with requests_mock.Mocker() as req_mock: - req_mock.get( - 'https://trunk.rdoproject.org/centos7-master/tripleo-ci-testing/commit.yaml', # noqa - text=test_fakes.TEST_COMMIT_YAML_CENTOS_7, - ) - with mock.patch('os.path.isfile') as mock_is_file: - mock_is_file.return_value = False - self.assertRaises( - exc.TripleOHashMissingConfig, - thi.TripleOHashInfo, - 'centos7', - 'master', - None, - 'tripleo-ci-testing', - ) - def test_override_config_dlrn_url(self, mock_config): expected_dlrn_url = 'https://foo.bar.baz/centos8-master/component/common/current-tripleo/commit.yaml' # noqa - with requests_mock.Mocker() as req_mock: - req_mock.get( - expected_dlrn_url, - text=test_fakes.TEST_COMMIT_YAML_COMPONENT, - ) + mocked = MagicMock( + return_value=(test_fakes.TEST_COMMIT_YAML_COMPONENT, 200)) + with patch( + 'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked): mock_hash_info = thi.TripleOHashInfo( 'centos8', 'master', 'common', 'current-tripleo', {'dlrn_url': 'https://foo.bar.baz'} @@ -220,11 +187,10 @@ class TestGetHashInfo(unittest.TestCase): def test_override_config_dlrn_url_empty_ignored(self, mock_config): expected_dlrn_url = 'https://trunk.rdoproject.org/centos8-master/component/common/current-tripleo/commit.yaml' # noqa - with requests_mock.Mocker() as req_mock: - req_mock.get( - expected_dlrn_url, - text=test_fakes.TEST_COMMIT_YAML_COMPONENT, - ) + mocked = MagicMock( + return_value=(test_fakes.TEST_COMMIT_YAML_COMPONENT, 200)) + with patch( + 'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked): mock_hash_info = thi.TripleOHashInfo( 'centos8', 'master', 'common', 'current-tripleo', {'dlrn_url': ''} diff --git a/tox.ini b/tox.ini index f28686f..14a9ba1 100644 --- a/tox.ini +++ b/tox.ini @@ -76,6 +76,7 @@ commands = # Ensure that ansible is able to load the modules, as syntax check will fail # if modules cannot be loaded. sh -c "ansible-playbook --syntax-check playbooks/*.yaml" + ansible localhost -m tripleo.repos.get_hash -a "release=master os_version=centos8" whitelist_externals = sh