From 50203ab44afa44f52c6f09d5450cb1689eb01a4f Mon Sep 17 00:00:00 2001 From: Gabriele Cerami Date: Fri, 28 Feb 2020 11:52:28 +0000 Subject: [PATCH] Add centos8 support for emit_releases_script emit_releases_file needed to be updated to support centos8 and integration pipeline. I needed to make the script distro aware, then distringuish between distros to get the proper repo and hash in get_dlrn_hash function. The tests have been only partially updated: I updated the success script to cover the centos8 case, and accountfor the change in signature for get_dlrn_hash, but the rest of the workflow is exaclty the same either if we are in centos7 or centos8, so I did not add any additional checks. Change-Id: I26799d5e6a3ed404b4a29be621c5871ef9be6e4c --- roles/run-test/templates/toci_gate_test.sh.j2 | 2 + .../emit_releases_file/emit_releases_file.py | 118 +++++++++++++----- .../emit_releases_file/test_get_dlrn_hash.py | 23 +++- .../emit_releases_file/test_release_name.py | 5 +- .../test_write_bash_script.py | 2 +- tox.ini | 2 +- 6 files changed, 114 insertions(+), 38 deletions(-) diff --git a/roles/run-test/templates/toci_gate_test.sh.j2 b/roles/run-test/templates/toci_gate_test.sh.j2 index 91570b718..507ed1064 100755 --- a/roles/run-test/templates/toci_gate_test.sh.j2 +++ b/roles/run-test/templates/toci_gate_test.sh.j2 @@ -119,6 +119,8 @@ if [[ -f "$RELEASES_SCRIPT" ]] && [[ {{ featureset }} =~ 010|011|037|047|050|056 --featureset-file $TRIPLEO_ROOT/tripleo-quickstart/config/general_config/$(basename {{ job_featureset_file }}) \ --output-file $RELEASES_FILE_OUTPUT \ --log-file $RELEASES_SCRIPT_LOGFILE \ + --distro-name {{ ansible_distribution | lower }} \ + --distro-version {{ ansible_distribution_major_version }} \ $EMIT_RELEASES_EXTRA_ARGS fi fi diff --git a/scripts/emit_releases_file/emit_releases_file.py b/scripts/emit_releases_file/emit_releases_file.py index 31653213b..492dce236 100644 --- a/scripts/emit_releases_file/emit_releases_file.py +++ b/scripts/emit_releases_file/emit_releases_file.py @@ -81,19 +81,29 @@ def load_featureset_file(featureset_file): return featureset -def get_dlrn_hash(release, hash_name, retries=20, timeout=8): +def get_dlrn_hash(release, hash_name, distro_name='centos', distro_version='7', + retries=20, timeout=8): """Get the dlrn hash for the release and hash name Retrieves the delorean.repo for the provided release and hash name, e.g. https://trunk.rdoproject.org/centos7-master/current/delorean.repo for master and current. The hash is taken from the repo file contents and returned. + :param distro_name: Distro name + :param distro_version: Distro version """ logger = logging.getLogger('emit-releases') full_hash_pattern = re.compile('[a-z,0-9]{40}_[a-z,0-9]{8}') rdo_url = os.getenv('NODEPOOL_RDO_PROXY', 'https://trunk.rdoproject.org') - repo_url = ('%s/centos7-%s/%s/delorean.repo' % - (rdo_url, release, hash_name)) + logger.error("distro %s version %s", distro_name, distro_version) + if distro_name == 'centos' and distro_version == '7': + repo_url = ('%s/centos7-%s/%s/delorean.repo' % + (rdo_url, release, hash_name)) + elif distro_name == 'centos' and distro_version == '8': + repo_url = ('%s/centos8-%s/%s/delorean.repo.md5' % + (rdo_url, release, hash_name)) + logger.debug("distro_name {} distro_version {} repo_url {}" + "".format(distro_name, distro_version, repo_url)) for retry_num in range(retries): repo_file = None try: @@ -102,33 +112,37 @@ def get_dlrn_hash(release, hash_name, retries=20, timeout=8): logger.warning("Attempt {} of {} to get DLRN hash threw an " "exception.".format(retry_num + 1, retries)) logger.exception(e) - pass - else: - if repo_file is not None and repo_file.ok: - break + continue + if repo_file is not None and repo_file.ok: + if distro_name == 'centos' and distro_version == '7': + print(repo_file.text) + full_hash = full_hash_pattern.findall(repo_file.text)[0] + elif distro_name == 'centos' and distro_version == '8': + full_hash = repo_file.text + break - elif repo_file: - logger.warning("Attempt {} of {} to get DLRN hash returned " - "status code {}.".format(retry_num + 1, - retries, - repo_file.status_code)) - else: - logger.warning("Attempt {} of {} to get DLRN hash failed to " - "get a response.".format(retry_num + 1, - retries)) + elif repo_file: + logger.warning("Attempt {} of {} to get DLRN hash returned " + "status code {}.".format(retry_num + 1, + retries, + repo_file.status_code)) + else: + logger.warning("Attempt {} of {} to get DLRN hash failed to " + "get a response.".format(retry_num + 1, + retries)) if repo_file is None or not repo_file.ok: raise RuntimeError("Failed to retrieve repo file from {} after " "{} retries".format(repo_url, retries)) - full_hash = full_hash_pattern.findall(repo_file.text) logger.info("Got DLRN hash: {} for the named hash: {} on the {} " - "release".format(full_hash[0], hash_name, release)) - return full_hash[0] + "release".format(full_hash, hash_name, release)) + return full_hash def compose_releases_dictionary(stable_release, featureset, upgrade_from, - is_periodic=False): + is_periodic=False, + distro_name='centos', distro_version='7'): """Compose the release dictionary for stable_release and featureset This contains the main logic determining the contents of the release file. @@ -148,6 +162,8 @@ def compose_releases_dictionary(stable_release, featureset, upgrade_from, rocky). * Overcloud Update: stable_release is equal to target, but going from previous-current-tripleo to current hashes. + :param distro_name: Distro name + :param distro_version: Distro version """ logger = logging.getLogger('emit-releases') if stable_release not in RELEASES: @@ -186,13 +202,21 @@ def compose_releases_dictionary(stable_release, featureset, upgrade_from, "used in a fast forward upgrade. Current long-term support " "releases: {}".format(stable_release, LONG_TERM_SUPPORT_RELEASES)) - newest_hash = get_dlrn_hash(stable_release, NEWEST_HASH_NAME) + newest_hash = get_dlrn_hash(stable_release, NEWEST_HASH_NAME, + distro_name, + distro_version) if stable_release == 'newton': - current_hash = get_dlrn_hash(stable_release, NEWTON_HASH_NAME) + current_hash = get_dlrn_hash(stable_release, NEWTON_HASH_NAME, + distro_name, + distro_version) elif is_periodic: - current_hash = get_dlrn_hash(stable_release, PROMOTION_HASH_NAME) + current_hash = get_dlrn_hash(stable_release, PROMOTION_HASH_NAME, + distro_name, + distro_version) else: - current_hash = get_dlrn_hash(stable_release, CURRENT_HASH_NAME) + current_hash = get_dlrn_hash(stable_release, CURRENT_HASH_NAME, + distro_name, + distro_version) releases_dictionary = { 'undercloud_install_release': stable_release, @@ -216,9 +240,13 @@ def compose_releases_dictionary(stable_release, featureset, upgrade_from, logger.info('Doing an overcloud upgrade') deploy_release = get_relative_release(stable_release, -1) if deploy_release == 'newton': - deploy_hash = get_dlrn_hash(deploy_release, NEWTON_HASH_NAME) + deploy_hash = get_dlrn_hash(deploy_release, NEWTON_HASH_NAME, + distro_name, + distro_version) else: - deploy_hash = get_dlrn_hash(deploy_release, CURRENT_HASH_NAME) + deploy_hash = get_dlrn_hash(deploy_release, CURRENT_HASH_NAME, + distro_name, + distro_version) releases_dictionary['overcloud_deploy_release'] = deploy_release releases_dictionary['overcloud_deploy_hash'] = deploy_hash @@ -226,24 +254,34 @@ def compose_releases_dictionary(stable_release, featureset, upgrade_from, logger.info('Doing an overcloud fast forward upgrade') deploy_release = get_relative_release(stable_release, -3) if deploy_release == 'newton': - deploy_hash = get_dlrn_hash(deploy_release, NEWTON_HASH_NAME) + deploy_hash = get_dlrn_hash(deploy_release, NEWTON_HASH_NAME, + distro_name, + distro_version) else: - deploy_hash = get_dlrn_hash(deploy_release, CURRENT_HASH_NAME) + deploy_hash = get_dlrn_hash(deploy_release, CURRENT_HASH_NAME, + distro_name, + distro_version) releases_dictionary['overcloud_deploy_release'] = deploy_release releases_dictionary['overcloud_deploy_hash'] = deploy_hash elif featureset.get('undercloud_upgrade'): logger.info('Doing an undercloud upgrade') install_release = get_relative_release(stable_release, -1) - install_hash = get_dlrn_hash(install_release, CURRENT_HASH_NAME) + install_hash = get_dlrn_hash(install_release, CURRENT_HASH_NAME, + distro_name, + distro_version) releases_dictionary['undercloud_install_release'] = install_release releases_dictionary['undercloud_install_hash'] = install_hash elif featureset.get('standalone_upgrade'): logger.info('Doing an standalone upgrade') install_release = get_relative_release(stable_release, -1) - install_hash = get_dlrn_hash(install_release, CURRENT_HASH_NAME) - install_newest_hash = get_dlrn_hash(install_release, NEWEST_HASH_NAME) + install_hash = get_dlrn_hash(install_release, CURRENT_HASH_NAME, + distro_name, + distro_version) + install_newest_hash = get_dlrn_hash(install_release, NEWEST_HASH_NAME, + distro_name, + distro_version) releases_dictionary['standalone_deploy_release'] = install_release releases_dictionary['standalone_deploy_newest_hash'] = \ install_newest_hash @@ -251,7 +289,9 @@ def compose_releases_dictionary(stable_release, featureset, upgrade_from, elif featureset.get('overcloud_update'): logger.info('Doing an overcloud update') - previous_hash = get_dlrn_hash(stable_release, PREVIOUS_HASH_NAME) + previous_hash = get_dlrn_hash(stable_release, PREVIOUS_HASH_NAME, + distro_name, + distro_version) releases_dictionary['overcloud_deploy_hash'] = previous_hash logger.debug("stable_release: %s, featureset: %s", stable_release, @@ -347,6 +387,16 @@ if __name__ == '__main__': help='Release that the change being tested is from.\n' 'All other releases are calculated from this\n' 'basis.') + parser.add_argument( + '--distro-name', + choices=['centos'], + required=True, + help='Distribution name') + parser.add_argument( + '--distro-version', + choices=['7', '8'], + required=True, + help='Distribution version') parser.add_argument( '--featureset-file', required=True, @@ -381,7 +431,9 @@ if __name__ == '__main__': releases_dictionary = compose_releases_dictionary(args.stable_release, featureset, args.upgrade_from, - args.is_periodic) + args.is_periodic, + args.distro_name, + args.distro_version) releases_dictionary = shim_convert_old_release_names(releases_dictionary, args.is_periodic) diff --git a/scripts/emit_releases_file/test_get_dlrn_hash.py b/scripts/emit_releases_file/test_get_dlrn_hash.py index 05b1c0e90..73ea60406 100644 --- a/scripts/emit_releases_file/test_get_dlrn_hash.py +++ b/scripts/emit_releases_file/test_get_dlrn_hash.py @@ -29,7 +29,28 @@ def test_get_dlrn_hash_ok(mock_get, mock_logging): dlrn_hash = '81c23c047e8e0fc03b54164921f49fdb4103202c_b333f915' repo_url = ('https://trunk.rdoproject.org/centos7-%s/%s/delorean.repo' % (release, hash_name)) - assert get_dlrn_hash(release, hash_name) == dlrn_hash + assert get_dlrn_hash(release, hash_name, distro_name='centos', + distro_version='7') == dlrn_hash + mock_get.assert_called_once_with(repo_url, timeout=8) + mock_log_info.assert_called_once_with("Got DLRN hash: {} for the named " + "hash: {} on the {} " + "release".format(dlrn_hash, + hash_name, + release)) + mock_log_warning.assert_not_called() + mock_log_exception.assert_not_called() + + mock_get.reset_mock() + mock_log_info.reset_mock() + mock_response.text = '7e8e0fc03b54164921f49fdb4103202c' + mock_get.return_value = mock_response + release = 'master' + hash_name = 'current-tripleo' + dlrn_hash = '7e8e0fc03b54164921f49fdb4103202c' + repo_url = ('https://trunk.rdoproject.org/centos8-%s/%s/delorean.repo.md5' % + (release, hash_name)) + assert get_dlrn_hash(release, hash_name, distro_name='centos', + distro_version='8') == dlrn_hash mock_get.assert_called_once_with(repo_url, timeout=8) mock_log_info.assert_called_once_with("Got DLRN hash: {} for the named " "hash: {} on the {} " diff --git a/scripts/emit_releases_file/test_release_name.py b/scripts/emit_releases_file/test_release_name.py index dd352d8ce..13463b342 100644 --- a/scripts/emit_releases_file/test_release_name.py +++ b/scripts/emit_releases_file/test_release_name.py @@ -15,11 +15,12 @@ def hash_mock_setup(): calls_args = [] def _hash_mock_setup(get_hash_mock, calls): - get_hash_mock.side_effect = lambda r, h: calls[(r, h)] + get_hash_mock.side_effect = lambda r, h, t, s : calls[(r, h)] # Store the references to use them at tear down setup_mock.append(get_hash_mock) - calls_args.append([mock.call(*cargs) for cargs in calls]) + calls_args.append([mock.call(cargs[0], cargs[1], 'centos', '7') for + cargs in calls]) yield _hash_mock_setup diff --git a/scripts/emit_releases_file/test_write_bash_script.py b/scripts/emit_releases_file/test_write_bash_script.py index 9c204d875..f09d8838f 100644 --- a/scripts/emit_releases_file/test_write_bash_script.py +++ b/scripts/emit_releases_file/test_write_bash_script.py @@ -13,7 +13,7 @@ else: BUILTINS_OPEN = "builtins.open" -def test_empty_relases_dictionary_fails(): +def test_empty_releases_dictionary_fails(): assert (not write_releases_dictionary_to_bash({}, "")) diff --git a/tox.ini b/tox.ini index 78b2e5da0..91a0a3e02 100644 --- a/tox.ini +++ b/tox.ini @@ -35,7 +35,7 @@ commands_pre = pip install -q bindep bindep test commands = - python -m pytest --html={envlogdir}/reports.html --self-contained-html {posargs:--cov=emit_releases_file} scripts + python -m pytest --html={envlogdir}/reports.html --self-contained-html --cov=emit_releases_file {posargs:scripts} [testenv:venv] commands = {posargs}