fuel-qa/fuelweb_test/helpers/replace_repos.py
Sergey Kulanov 3990d8a34f Fix repos list processing flow
We must not modify lists in `for` loops because one builds a
completely new list and then gives it the same name the old list
has [1]

[1]. https://docs.python.org/2/reference/compound_stmts.html#for

Change-Id: Id77ed7814701d4cac83c4c9d72c667a839ed05f9
Closes-bug: #1562530
2016-04-05 08:03:11 +00:00

239 lines
9.4 KiB
Python

# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import re
from fuelweb_test import logger
import fuelweb_test.settings as help_data
def replace_ubuntu_repos(repos_attr, upstream_host):
# Walk thru repos_attr and replace/add extra Ubuntu mirrors
if help_data.MIRROR_UBUNTU:
logger.debug("Adding new mirrors: '{0}'"
.format(help_data.MIRROR_UBUNTU))
repos = add_ubuntu_mirrors()
# Keep other (not upstream) repos, skip previously added ones
for repo_value in repos_attr['value']:
if upstream_host not in repo_value['uri']:
if check_new_ubuntu_repo(repos, repo_value):
repos.append(repo_value)
else:
logger.debug("Removing mirror: '{0} {1}'"
.format(repo_value['name'], repo_value['uri']))
else:
# Use defaults from Nailgun if MIRROR_UBUNTU is not set
repos = repos_attr['value']
if help_data.EXTRA_DEB_REPOS:
repos = add_ubuntu_extra_mirrors(repos=repos)
if help_data.PATCHING_DISABLE_UPDATES:
repos = [repo for repo in repos if repo['name']
not in ('mos-updates', 'mos-security')]
return repos
def replace_centos_repos(repos_attr, upstream_host):
# Walk thru repos_attr and replace/add extra Centos mirrors
if help_data.MIRROR_CENTOS:
logger.debug("Adding new mirrors: '{0}'"
.format(help_data.MIRROR_CENTOS))
repos = add_centos_mirrors()
# Keep other (not upstream) repos, skip previously added ones
for repo_value in repos_attr['value']:
# self.admin_node_ip while repo is located on master node
if upstream_host not in repo_value['uri']:
if check_new_centos_repo(repos, repo_value):
repos.append(repo_value)
else:
logger.debug("Removing mirror: '{0} {1}'"
.format(repo_value['name'], repo_value['uri']))
else:
# Use defaults from Nailgun if MIRROR_CENTOS is not set
repos = repos_attr['value']
if help_data.EXTRA_RPM_REPOS:
repos = add_centos_extra_mirrors(repos=repos)
if help_data.PATCHING_DISABLE_UPDATES:
repos = [repo for repo in repos if repo['name']
not in ('mos-updates', 'mos-security')]
return repos
def report_repos(repos_attr, release=help_data.OPENSTACK_RELEASE):
"""Show list of repositories for specified cluster"""
if help_data.OPENSTACK_RELEASE_UBUNTU in release:
report_ubuntu_repos(repos_attr['value'])
else:
report_centos_repos(repos_attr['value'])
def report_ubuntu_repos(repos):
for x, rep in enumerate(repos):
logger.info(
"Ubuntu repo {0} '{1}': '{2} {3} {4} {5}', priority:{6}"
.format(x, rep['name'], rep['type'], rep['uri'],
rep['suite'], rep['section'], rep['priority']))
def report_centos_repos(repos):
for x, rep in enumerate(repos):
logger.info(
"Centos repo {0} '{1}': '{2} {3}', priority:{4}"
.format(x, rep['name'], rep['type'], rep['uri'],
rep['priority']))
def add_ubuntu_mirrors(repos=None, mirrors=help_data.MIRROR_UBUNTU,
priority=help_data.MIRROR_UBUNTU_PRIORITY):
if not repos:
repos = []
# Add external Ubuntu repositories
for x, repo_str in enumerate(mirrors.split('|')):
repo_value = parse_ubuntu_repo(
repo_str, 'ubuntu-{0}'.format(x), priority)
if repo_value and check_new_ubuntu_repo(repos, repo_value):
repos.append(repo_value)
return repos
def add_centos_mirrors(repos=None, mirrors=help_data.MIRROR_CENTOS,
priority=help_data.MIRROR_CENTOS_PRIORITY):
if not repos:
repos = []
# Add external Centos repositories
for repo_str in mirrors.split('|'):
repo_value = parse_centos_repo(repo_str, priority)
if repo_value and check_new_centos_repo(repos, repo_value):
repos.append(repo_value)
return repos
def add_ubuntu_extra_mirrors(repos=None, prefix='extra',
mirrors=help_data.EXTRA_DEB_REPOS,
priority=help_data.EXTRA_DEB_REPOS_PRIORITY):
if not repos:
repos = []
# Add extra Ubuntu repositories with higher priority
for x, repo_str in enumerate(mirrors.split('|')):
repo_value = parse_ubuntu_repo(
repo_str, '{0}-{1}'.format(prefix, x), priority)
if repo_value and check_new_ubuntu_repo(repos, repo_value):
# Remove repos that use the same name
repos = [repo for repo in repos
if repo["name"] != repo_value["name"]]
repos.append(repo_value)
return repos
def add_centos_extra_mirrors(repos=None,
mirrors=help_data.EXTRA_RPM_REPOS,
priority=help_data.EXTRA_RPM_REPOS_PRIORITY):
if not repos:
repos = []
# Add extra Centos repositories
for repo_str in mirrors.split('|'):
repo_value = parse_centos_repo(repo_str, priority)
if repo_value and check_new_centos_repo(repos, repo_value):
# Remove repos that use the same name
repos = [repo for repo in repos
if repo["name"] != repo_value["name"]]
repos.append(repo_value)
return repos
def check_new_ubuntu_repo(repos, repo_value):
# Checks that 'repo_value' is a new unique record for Ubuntu 'repos'
for repo in repos:
if (repo["type"] == repo_value["type"] and
repo["uri"] == repo_value["uri"] and
repo["suite"] == repo_value["suite"] and
repo["section"] == repo_value["section"]):
return False
return True
def check_new_centos_repo(repos, repo_value):
# Checks that 'repo_value' is a new unique record for Centos 'repos'
for repo in repos:
if repo["uri"] == repo_value["uri"]:
return False
return True
def parse_ubuntu_repo(repo_string, name, priority):
# Validate DEB repository string format
results = re.search("""
^ # [beginning of the string]
([\w\-\.\/]+)? # group 1: optional repository name (for Nailgun)
,? # [optional comma separator]
(deb|deb-src) # group 2: type; search for 'deb' or 'deb-src'
\s+ # [space separator]
( # group 3: uri;
\w+:\/\/ # - protocol, i.e. 'http://'
[\w\-\.\/]+ # - hostname
(?::\d+) # - port, i.e. ':8080', if exists
?[\w\-\.\/]+ # - rest of the path, if exists
) # - end of group 2
\s+ # [space separator]
([\w\-\.\/]+) # group 4: suite;
\s* # [space separator], if exists
( # group 5: section;
[\w\-\.\/\s]* # - several space-separated names, or None
) # - end of group 4
,? # [optional comma separator]
(\d+)? # group 6: optional priority of the repository
$ # [ending of the string]""",
repo_string.strip(), re.VERBOSE)
if results:
return {"name": results.group(1) or name,
"priority": int(results.group(6) or priority),
"type": results.group(2),
"uri": results.group(3),
"suite": results.group(4),
"section": results.group(5) or ''}
else:
logger.error("Provided DEB repository has incorrect format: {}"
.format(repo_string))
def parse_centos_repo(repo_string, priority):
# Validate RPM repository string format
results = re.search("""
^ # [beginning of the string]
([\w\-\.\/]+) # group 1: repo name
, # [comma separator]
( # group 2: uri;
\w+:\/\/ # - protocol, i.e. 'http://'
[\w\-\.\/]+ # - hostname
(?::\d+) # - port, i.e. ':8080', if exists
?[\w\-\.\/]+ # - rest of the path, if exists
) # - end of group 2
\s* # [space separator]
,? # [optional comma separator]
(\d+)? # group 3: optional priority of the repository
$ # [ending of the string]""",
repo_string.strip(), re.VERBOSE)
if results:
return {"name": results.group(1),
"priority": int(results.group(3) or priority),
"type": 'rpm',
"uri": results.group(2)}
else:
logger.error("Provided RPM repository has incorrect format: {}"
.format(repo_string))