Replace Ubuntu mirror in /etc/fuel/astute.yaml to local mirror
- Use the local Ubuntu mirror to speed building the Ubuntu bootstrap image, - Move replace_repos modules to a separate module. Change-Id: I263752884a7b629a76f5ce7cb3e921390c11af90 Closes-Bug:#1487498
This commit is contained in:
Dennis Dmitriev
parent
c2cb46ff09
commit
00dde4b1ba
|
|
@ -93,6 +93,11 @@ Regenerate Repo
|
|||
.. automodule:: fuelweb_test.helpers.regenerate_repo
|
||||
:members:
|
||||
|
||||
Replace Reposiroties
|
||||
--------------------
|
||||
.. automodule:: fuelweb_test.helpers.replace_repos
|
||||
:members:
|
||||
|
||||
Security
|
||||
--------
|
||||
.. automodule:: fuelweb_test.helpers.security
|
||||
|
|
|
|||
|
|
@ -29,9 +29,11 @@ from fuelweb_test.helpers import checkers
|
|||
|
||||
from fuelweb_test.helpers.regenerate_repo import regenerate_centos_repo
|
||||
from fuelweb_test.helpers.regenerate_repo import regenerate_ubuntu_repo
|
||||
from fuelweb_test.helpers import replace_repos
|
||||
from fuelweb_test.helpers.utils import cond_upload
|
||||
from fuelweb_test.settings import FUEL_PLUGIN_BUILDER_REPO
|
||||
from fuelweb_test.settings import FUEL_USE_LOCAL_NTPD
|
||||
from fuelweb_test.settings import MIRROR_UBUNTU
|
||||
from fuelweb_test import settings as hlp_data
|
||||
from fuelweb_test.settings import NESSUS_IMAGE_PATH
|
||||
|
||||
|
|
@ -108,7 +110,7 @@ class BaseActions(object):
|
|||
def wait_for_ready_container(self, timeout=300):
|
||||
wait(lambda: self.is_container_ready, timeout=timeout)
|
||||
|
||||
def change_content_in_yaml(self, old_file, new_file, element, value):
|
||||
def put_value_to_local_yaml(self, old_file, new_file, element, value):
|
||||
"""Changes content in old_file at element is given to the new value
|
||||
and creates new file with changed content
|
||||
:param old_file: a path to the file content from to be changed
|
||||
|
|
@ -129,7 +131,32 @@ class BaseActions(object):
|
|||
yaml_dict[element[-1]] = value
|
||||
|
||||
with open(new_file, 'w') as f_new:
|
||||
yaml.dump(origin_yaml, f_new)
|
||||
yaml.dump(origin_yaml, f_new, default_flow_style=False)
|
||||
|
||||
def get_value_from_local_yaml(self, yaml_file, element):
|
||||
"""Get a value of the element from the local yaml file
|
||||
|
||||
:param str yaml_file: a path to the yaml file
|
||||
:param list element:
|
||||
list with path to element to be read
|
||||
for example: ['root_elem', 'first_elem', 'target_elem']
|
||||
if there are a few elements with equal names use integer
|
||||
to identify which element should be used
|
||||
:return obj: value
|
||||
"""
|
||||
with open(yaml_file, 'r') as f_old:
|
||||
yaml_dict = yaml.load(f_old)
|
||||
|
||||
for i, k in enumerate(element):
|
||||
try:
|
||||
yaml_dict = yaml_dict[k]
|
||||
except IndexError:
|
||||
raise IndexError("Element {0} not found in the file {1}"
|
||||
.format(element[: i + 1], f_old))
|
||||
except KeyError:
|
||||
raise KeyError("Element {0} not found in the file {1}"
|
||||
.format(element[: i + 1], f_old))
|
||||
return yaml_dict
|
||||
|
||||
def change_yaml_file_in_container(
|
||||
self, path_to_file, element, value, container=None):
|
||||
|
|
@ -144,16 +171,69 @@ class BaseActions(object):
|
|||
if not container:
|
||||
container = self.container
|
||||
|
||||
old_file = '/tmp/temp_file.old.yaml'
|
||||
new_file = '/tmp/temp_file.new.yaml'
|
||||
old_file = '/tmp/temp_file_{0}.old.yaml'.format(str(os.getpid()))
|
||||
new_file = '/tmp/temp_file_{0}.new.yaml'.format(str(os.getpid()))
|
||||
|
||||
self.copy_between_node_and_container(
|
||||
'{0}:{1}'.format(container, path_to_file), old_file)
|
||||
self.admin_remote.download(old_file, old_file)
|
||||
self.change_content_in_yaml(old_file, new_file, element, value)
|
||||
self.put_value_to_local_yaml(old_file, new_file, element, value)
|
||||
self.admin_remote.upload(new_file, new_file)
|
||||
self.copy_between_node_and_container(
|
||||
new_file, '{0}:{1}'.format(container, path_to_file))
|
||||
os.remove(old_file)
|
||||
os.remove(new_file)
|
||||
|
||||
def get_value_from_yaml(self, path_to_file, element):
|
||||
"""Get a value from the yaml file stored in container
|
||||
or on master node if self.container is None
|
||||
|
||||
:param str path_to_file: absolutely path to the file
|
||||
:param list element: list with path to the element be changed
|
||||
:return obj: value
|
||||
"""
|
||||
|
||||
if self.container:
|
||||
admin_tmp_file = '/tmp/temp_file_{0}.yaml'.format(str(os.getpid()))
|
||||
self.copy_between_node_and_container(
|
||||
'{0}:{1}'.format(self.container, path_to_file), admin_tmp_file)
|
||||
else:
|
||||
admin_tmp_file = path_to_file
|
||||
|
||||
host_tmp_file = '/tmp/temp_file_{0}.yaml'.format(str(os.getpid()))
|
||||
|
||||
self.admin_remote.download(admin_tmp_file, host_tmp_file)
|
||||
value = self.get_value_from_local_yaml(host_tmp_file, element)
|
||||
os.remove(host_tmp_file)
|
||||
return value
|
||||
|
||||
def put_value_to_yaml(self, path_to_file, element, value):
|
||||
"""Put a value to the yaml file stored in container
|
||||
or on master node if self.container is None
|
||||
|
||||
:param str path_to_file: absolutely path to the file
|
||||
:param list element: list with path to the element be changed
|
||||
:param value: new value for element
|
||||
:return: None
|
||||
"""
|
||||
|
||||
if self.container:
|
||||
admin_tmp_file = '/tmp/temp_file_{0}.yaml'.format(str(os.getpid()))
|
||||
self.copy_between_node_and_container(
|
||||
'{0}:{1}'.format(self.container, path_to_file), admin_tmp_file)
|
||||
else:
|
||||
admin_tmp_file = path_to_file
|
||||
|
||||
host_tmp_file = '/tmp/temp_file_{0}.yaml'.format(str(os.getpid()))
|
||||
|
||||
self.admin_remote.download(admin_tmp_file, host_tmp_file)
|
||||
self.put_value_to_local_yaml(host_tmp_file, host_tmp_file,
|
||||
element, value)
|
||||
self.admin_remote.upload(host_tmp_file, admin_tmp_file)
|
||||
if self.container:
|
||||
self.copy_between_node_and_container(
|
||||
admin_tmp_file, '{0}:{1}'.format(self.container, path_to_file))
|
||||
os.remove(host_tmp_file)
|
||||
|
||||
|
||||
class AdminActions(BaseActions):
|
||||
|
|
@ -202,6 +282,15 @@ class AdminActions(BaseActions):
|
|||
assert_equal(0, result['exit_code'],
|
||||
"Command [{cmd}] failed with the following "
|
||||
"result: {res}".format(cmd=cmd, res=result))
|
||||
if MIRROR_UBUNTU:
|
||||
logger.info("Replace Ubuntu mirror for bootstrap image in {0}"
|
||||
.format(config))
|
||||
repo_url = self.get_value_from_yaml(config, ['BOOTSTRAP',
|
||||
'MIRROR_DISTRO'])
|
||||
new_repo_url = replace_repos.replace_ubuntu_repo_url(
|
||||
repo_url, upstream_host='archive.ubuntu.com')
|
||||
self.put_value_to_yaml(config, ['BOOTSTRAP', 'MIRROR_DISTRO'],
|
||||
new_repo_url)
|
||||
|
||||
@logwrap
|
||||
def upload_packages(self, local_packages_dir, centos_repo_path,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,262 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import re
|
||||
|
||||
from fuelweb_test import logger
|
||||
import fuelweb_test.settings as help_data
|
||||
|
||||
|
||||
def replace_ubuntu_repo_url(repo_url, upstream_host):
|
||||
repos_attr = {
|
||||
"value": [
|
||||
{
|
||||
"name": "ubuntu-bootstrap",
|
||||
"uri": repo_url,
|
||||
"type": "deb",
|
||||
"section": None,
|
||||
"suite": None
|
||||
}
|
||||
]
|
||||
}
|
||||
repos = replace_ubuntu_repos(repos_attr, upstream_host)
|
||||
new_repo_url = repos[0]['uri']
|
||||
if new_repo_url != repo_url:
|
||||
logger.debug("Ubuntu repository url changed from '{0}' to '{1}'"
|
||||
.format(repo_url, new_repo_url))
|
||||
return new_repo_url
|
||||
|
||||
|
||||
def replace_ubuntu_repos(repos_attr, upstream_host):
|
||||
# Walk thru repos_attr and replace/add extra Ubuntu mirrors
|
||||
repos = []
|
||||
if help_data.MIRROR_UBUNTU:
|
||||
logger.debug("Adding new mirrors: '{0}'"
|
||||
.format(help_data.MIRROR_UBUNTU))
|
||||
repos = add_ubuntu_mirrors()
|
||||
# Keep other (not upstream) repos, skip previously added ones
|
||||
for repo_value in repos_attr['value']:
|
||||
if upstream_host not in repo_value['uri']:
|
||||
if check_new_ubuntu_repo(repos, repo_value):
|
||||
repos.append(repo_value)
|
||||
else:
|
||||
logger.debug("Removing mirror: '{0} {1}'"
|
||||
.format(repo_value['name'], repo_value['uri']))
|
||||
else:
|
||||
# Use defaults from Nailgun if MIRROR_UBUNTU is not set
|
||||
repos = repos_attr['value']
|
||||
if help_data.EXTRA_DEB_REPOS:
|
||||
repos = add_ubuntu_extra_mirrors(repos=repos)
|
||||
if help_data.PATCHING_DISABLE_UPDATES:
|
||||
for repo in repos:
|
||||
if repo['name'] in ('mos-updates', 'mos-security'):
|
||||
repos.remove(repo)
|
||||
|
||||
return repos
|
||||
|
||||
|
||||
def replace_centos_repos(repos_attr, upstream_host):
|
||||
# Walk thru repos_attr and replace/add extra Centos mirrors
|
||||
repos = []
|
||||
if help_data.MIRROR_CENTOS:
|
||||
logger.debug("Adding new mirrors: '{0}'"
|
||||
.format(help_data.MIRROR_CENTOS))
|
||||
repos = add_centos_mirrors()
|
||||
# Keep other (not upstream) repos, skip previously added ones
|
||||
for repo_value in repos_attr['value']:
|
||||
# self.admin_node_ip while repo is located on master node
|
||||
if upstream_host not in repo_value['uri']:
|
||||
if check_new_centos_repo(repos, repo_value):
|
||||
repos.append(repo_value)
|
||||
else:
|
||||
logger.debug("Removing mirror: '{0} {1}'"
|
||||
.format(repo_value['name'], repo_value['uri']))
|
||||
else:
|
||||
# Use defaults from Nailgun if MIRROR_CENTOS is not set
|
||||
repos = repos_attr['value']
|
||||
if help_data.EXTRA_RPM_REPOS:
|
||||
repos = add_centos_extra_mirrors(repos=repos)
|
||||
if help_data.PATCHING_DISABLE_UPDATES:
|
||||
for repo in repos:
|
||||
if repo['name'] in ('mos-updates', 'mos-security'):
|
||||
repos.remove(repo)
|
||||
|
||||
return repos
|
||||
|
||||
|
||||
def report_repos(repos_attr, release=help_data.OPENSTACK_RELEASE):
|
||||
"""Show list of reposifories for specified cluster"""
|
||||
if help_data.OPENSTACK_RELEASE_UBUNTU in release:
|
||||
report_ubuntu_repos(repos_attr['value'])
|
||||
else:
|
||||
report_centos_repos(repos_attr['value'])
|
||||
|
||||
|
||||
def report_ubuntu_repos(repos):
|
||||
for x, rep in enumerate(repos):
|
||||
logger.info(
|
||||
"Ubuntu repo {0} '{1}': '{2} {3} {4} {5}', priority:{6}"
|
||||
.format(x, rep['name'], rep['type'], rep['uri'],
|
||||
rep['suite'], rep['section'], rep['priority']))
|
||||
|
||||
|
||||
def report_centos_repos(repos):
|
||||
for x, rep in enumerate(repos):
|
||||
logger.info(
|
||||
"Centos repo {0} '{1}': '{2} {3}', priority:{4}"
|
||||
.format(x, rep['name'], rep['type'], rep['uri'],
|
||||
rep['priority']))
|
||||
|
||||
|
||||
def add_ubuntu_mirrors(repos=None, mirrors=help_data.MIRROR_UBUNTU,
|
||||
priority=help_data.MIRROR_UBUNTU_PRIORITY):
|
||||
if not repos:
|
||||
repos = []
|
||||
# Add external Ubuntu repositories
|
||||
for x, repo_str in enumerate(mirrors.split('|')):
|
||||
repo_value = parse_ubuntu_repo(
|
||||
repo_str, 'ubuntu-{0}'.format(x), priority)
|
||||
if repo_value and check_new_ubuntu_repo(repos, repo_value):
|
||||
repos.append(repo_value)
|
||||
return repos
|
||||
|
||||
|
||||
def add_centos_mirrors(repos=None, mirrors=help_data.MIRROR_CENTOS,
|
||||
priority=help_data.MIRROR_CENTOS_PRIORITY):
|
||||
if not repos:
|
||||
repos = []
|
||||
# Add external Centos repositories
|
||||
for x, repo_str in enumerate(mirrors.split('|')):
|
||||
repo_value = parse_centos_repo(repo_str, priority)
|
||||
if repo_value and check_new_centos_repo(repos, repo_value):
|
||||
repos.append(repo_value)
|
||||
return repos
|
||||
|
||||
|
||||
def add_ubuntu_extra_mirrors(repos=None, prefix='extra',
|
||||
mirrors=help_data.EXTRA_DEB_REPOS,
|
||||
priority=help_data.EXTRA_DEB_REPOS_PRIORITY):
|
||||
if not repos:
|
||||
repos = []
|
||||
# Add extra Ubuntu repositories with higher priority
|
||||
for x, repo_str in enumerate(mirrors.split('|')):
|
||||
repo_value = parse_ubuntu_repo(
|
||||
repo_str, '{0}-{1}'.format(prefix, x), priority)
|
||||
|
||||
if repo_value and check_new_ubuntu_repo(repos, repo_value):
|
||||
# Remove repos that use the same name
|
||||
for repo in repos:
|
||||
if repo["name"] == repo_value["name"]:
|
||||
repos.remove(repo)
|
||||
repos.append(repo_value)
|
||||
return repos
|
||||
|
||||
|
||||
def add_centos_extra_mirrors(repos=None,
|
||||
mirrors=help_data.EXTRA_RPM_REPOS,
|
||||
priority=help_data.EXTRA_RPM_REPOS_PRIORITY):
|
||||
if not repos:
|
||||
repos = []
|
||||
# Add extra Centos repositories
|
||||
for x, repo_str in enumerate(mirrors.split('|')):
|
||||
repo_value = parse_centos_repo(repo_str, priority)
|
||||
if repo_value and check_new_centos_repo(repos, repo_value):
|
||||
# Remove repos that use the same name
|
||||
for repo in repos:
|
||||
if repo["name"] == repo_value["name"]:
|
||||
repos.remove(repo)
|
||||
repos.append(repo_value)
|
||||
return repos
|
||||
|
||||
|
||||
def check_new_ubuntu_repo(repos, repo_value):
|
||||
# Checks that 'repo_value' is a new unique record for Ubuntu 'repos'
|
||||
for repo in repos:
|
||||
if (repo["type"] == repo_value["type"] and
|
||||
repo["uri"] == repo_value["uri"] and
|
||||
repo["suite"] == repo_value["suite"] and
|
||||
repo["section"] == repo_value["section"]):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def check_new_centos_repo(repos, repo_value):
|
||||
# Checks that 'repo_value' is a new unique record for Centos 'repos'
|
||||
for repo in repos:
|
||||
if repo["uri"] == repo_value["uri"]:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def parse_ubuntu_repo(repo_string, name, priority):
|
||||
# Validate DEB repository string format
|
||||
results = re.search("""
|
||||
^ # [beginning of the string]
|
||||
([\w\-\.\/]+)? # group 1: optional repository name (for Nailgun)
|
||||
,? # [optional comma separator]
|
||||
(deb|deb-src) # group 2: type; search for 'deb' or 'deb-src'
|
||||
\s+ # [space separator]
|
||||
( # group 3: uri;
|
||||
\w+:\/\/ # - protocol, i.e. 'http://'
|
||||
[\w\-\.\/]+ # - hostname
|
||||
(?::\d+) # - port, i.e. ':8080', if exists
|
||||
?[\w\-\.\/]+ # - rest of the path, if exists
|
||||
) # - end of group 2
|
||||
\s+ # [space separator]
|
||||
([\w\-\.\/]+) # group 4: suite;
|
||||
\s* # [space separator], if exists
|
||||
( # group 5: section;
|
||||
[\w\-\.\/\s]* # - several space-separated names, or None
|
||||
) # - end of group 4
|
||||
,? # [optional comma separator]
|
||||
(\d+)? # group 6: optional priority of the repository
|
||||
$ # [ending of the string]""",
|
||||
repo_string.strip(), re.VERBOSE)
|
||||
if results:
|
||||
return {"name": results.group(1) or name,
|
||||
"priority": int(results.group(6) or priority),
|
||||
"type": results.group(2),
|
||||
"uri": results.group(3),
|
||||
"suite": results.group(4),
|
||||
"section": results.group(5) or ''}
|
||||
else:
|
||||
logger.error("Provided DEB repository has incorrect format: {}"
|
||||
.format(repo_string))
|
||||
|
||||
|
||||
def parse_centos_repo(repo_string, priority):
|
||||
# Validate RPM repository string format
|
||||
results = re.search("""
|
||||
^ # [beginning of the string]
|
||||
([\w\-\.\/]+) # group 1: repo name
|
||||
, # [comma separator]
|
||||
( # group 2: uri;
|
||||
\w+:\/\/ # - protocol, i.e. 'http://'
|
||||
[\w\-\.\/]+ # - hostname
|
||||
(?::\d+) # - port, i.e. ':8080', if exists
|
||||
?[\w\-\.\/]+ # - rest of the path, if exists
|
||||
) # - end of group 2
|
||||
\s* # [space separator]
|
||||
,? # [optional comma separator]
|
||||
(\d+)? # group 3: optional priority of the repository
|
||||
$ # [ending of the string]""",
|
||||
repo_string.strip(), re.VERBOSE)
|
||||
if results:
|
||||
return {"name": results.group(1),
|
||||
"priority": int(results.group(3) or priority),
|
||||
"type": 'rpm',
|
||||
"uri": results.group(2)}
|
||||
else:
|
||||
logger.error("Provided RPM repository has incorrect format: {}"
|
||||
.format(repo_string))
|
||||
|
|
@ -42,6 +42,7 @@ from fuelweb_test.helpers.decorators import retry
|
|||
from fuelweb_test.helpers.decorators import update_fuel
|
||||
from fuelweb_test.helpers.decorators import update_ostf
|
||||
from fuelweb_test.helpers.decorators import upload_manifests
|
||||
from fuelweb_test.helpers import replace_repos
|
||||
from fuelweb_test.helpers.security import SecurityChecks
|
||||
from fuelweb_test.helpers.utils import run_on_remote
|
||||
from fuelweb_test import logger
|
||||
|
|
@ -592,15 +593,15 @@ class FuelWebClient(object):
|
|||
mirror = '{0},deb {1} {2} {3}'.format(name, mirror_url, suite, section)
|
||||
|
||||
attributes = self.client.get_cluster_attributes(cluster_id)
|
||||
|
||||
repos_attr = attributes['editable']['repo_setup']['repos']
|
||||
repos_attr['value'] = self.add_ubuntu_extra_mirrors(
|
||||
|
||||
repos_attr['value'] = replace_repos.add_ubuntu_extra_mirrors(
|
||||
repos=repos_attr['value'],
|
||||
prefix=suite,
|
||||
mirrors=mirror,
|
||||
priority=priority)
|
||||
|
||||
self.report_ubuntu_repos(repos_attr['value'])
|
||||
replace_repos.report_ubuntu_repos(repos_attr['value'])
|
||||
self.client.update_cluster_attributes(cluster_id, attributes)
|
||||
|
||||
def add_local_centos_mirror(self, cluster_id, name='Auxiliary',
|
||||
|
|
@ -613,18 +614,19 @@ class FuelWebClient(object):
|
|||
mirror = '{0},{1}'.format(repo_name, mirror_url)
|
||||
|
||||
attributes = self.client.get_cluster_attributes(cluster_id)
|
||||
|
||||
repos_attr = attributes['editable']['repo_setup']['repos']
|
||||
repos_attr['value'] = self.add_centos_extra_mirrors(
|
||||
|
||||
repos_attr['value'] = replace_repos.add_centos_extra_mirrors(
|
||||
repos=repos_attr['value'],
|
||||
mirrors=mirror,
|
||||
priority=priority)
|
||||
|
||||
self.report_centos_repos(repos_attr['value'])
|
||||
replace_repos.report_centos_repos(repos_attr['value'])
|
||||
self.client.update_cluster_attributes(cluster_id, attributes)
|
||||
|
||||
def replace_default_repos(self):
|
||||
# Replace Ubuntu default repositories for the release
|
||||
logger.info("Replace default repository list.")
|
||||
ubuntu_id = self.client.get_release_id(
|
||||
release_name=help_data.OPENSTACK_RELEASE_UBUNTU)
|
||||
|
||||
|
|
@ -632,8 +634,11 @@ class FuelWebClient(object):
|
|||
ubuntu_meta = ubuntu_release["attributes_metadata"]
|
||||
repos_ubuntu = ubuntu_meta["editable"]["repo_setup"]["repos"]
|
||||
|
||||
repos_ubuntu["value"] = self.replace_ubuntu_repos(repos_ubuntu)
|
||||
repos_ubuntu["value"] = replace_repos.replace_ubuntu_repos(
|
||||
repos_ubuntu, upstream_host='archive.ubuntu.com')
|
||||
|
||||
self.client.put_release(ubuntu_id, ubuntu_release)
|
||||
replace_repos.report_ubuntu_repos(repos_ubuntu["value"])
|
||||
|
||||
# Replace CentOS default repositories for the release
|
||||
centos_id = self.client.get_release_id(
|
||||
|
|
@ -643,224 +648,15 @@ class FuelWebClient(object):
|
|||
centos_meta = centos_release["attributes_metadata"]
|
||||
repos_centos = centos_meta["editable"]["repo_setup"]["repos"]
|
||||
|
||||
repos_centos["value"] = self.replace_centos_repos(repos_centos)
|
||||
repos_centos["value"] = replace_repos.replace_centos_repos(
|
||||
repos_centos, upstream_host=self.admin_node_ip)
|
||||
|
||||
self.client.put_release(centos_id, centos_release)
|
||||
replace_repos.report_centos_repos(repos_centos["value"])
|
||||
|
||||
def replace_ubuntu_repos(self, repos_attr):
|
||||
# Walk thru repos_attr and replace/add extra Ubuntu mirrors
|
||||
repos = []
|
||||
if help_data.MIRROR_UBUNTU:
|
||||
logger.info("Adding new mirrors: '{0}'"
|
||||
.format(help_data.MIRROR_UBUNTU))
|
||||
repos = self.add_ubuntu_mirrors()
|
||||
# Keep other (not upstream) repos, skip previously added ones
|
||||
for repo_value in repos_attr['value']:
|
||||
if 'archive.ubuntu.com' not in repo_value['uri']:
|
||||
if self.check_new_ubuntu_repo(repos, repo_value):
|
||||
repos.append(repo_value)
|
||||
else:
|
||||
logger.warning("Removing mirror: '{0}'"
|
||||
.format(repo_value['uri']))
|
||||
else:
|
||||
# Use defaults from Nailgun if MIRROR_UBUNTU is not set
|
||||
repos = repos_attr['value']
|
||||
if help_data.EXTRA_DEB_REPOS:
|
||||
repos = self.add_ubuntu_extra_mirrors(repos=repos)
|
||||
if help_data.PATCHING_DISABLE_UPDATES:
|
||||
for repo in repos:
|
||||
if repo['name'] in ('mos-updates', 'mos-security'):
|
||||
repos.remove(repo)
|
||||
|
||||
self.report_ubuntu_repos(repos)
|
||||
return repos
|
||||
|
||||
def replace_centos_repos(self, repos_attr):
|
||||
# Walk thru repos_attr and replace/add extra Centos mirrors
|
||||
repos = []
|
||||
if help_data.MIRROR_CENTOS:
|
||||
logger.info("Adding new mirrors: '{0}'"
|
||||
.format(help_data.MIRROR_CENTOS))
|
||||
repos = self.add_centos_mirrors()
|
||||
# Keep other (not upstream) repos, skip previously added ones
|
||||
for repo_value in repos_attr['value']:
|
||||
# self.admin_node_ip while repo is located on master node
|
||||
if self.admin_node_ip not in repo_value['uri']:
|
||||
if self.check_new_centos_repo(repos, repo_value):
|
||||
repos.append(repo_value)
|
||||
else:
|
||||
logger.warning("Removing mirror: '{0}'"
|
||||
.format(repo_value['uri']))
|
||||
else:
|
||||
# Use defaults from Nailgun if MIRROR_CENTOS is not set
|
||||
repos = repos_attr['value']
|
||||
if help_data.EXTRA_RPM_REPOS:
|
||||
repos = self.add_centos_extra_mirrors(repos=repos)
|
||||
if help_data.PATCHING_DISABLE_UPDATES:
|
||||
for repo in repos:
|
||||
if repo['name'] in ('mos-updates', 'mos-security'):
|
||||
repos.remove(repo)
|
||||
|
||||
self.report_centos_repos(repos)
|
||||
return repos
|
||||
|
||||
def report_repos(self, cluster_id, release=help_data.OPENSTACK_RELEASE):
|
||||
"""Show list of reposifories for specified cluster"""
|
||||
def get_cluster_repos(self, cluster_id):
|
||||
attributes = self.client.get_cluster_attributes(cluster_id)
|
||||
repos_attr = attributes['editable']['repo_setup']['repos']
|
||||
|
||||
if help_data.OPENSTACK_RELEASE_UBUNTU in release:
|
||||
self.report_ubuntu_repos(repos_attr['value'])
|
||||
else:
|
||||
self.report_centos_repos(repos_attr['value'])
|
||||
|
||||
def report_ubuntu_repos(self, repos):
|
||||
for x, rep in enumerate(repos):
|
||||
logger.info(
|
||||
"Ubuntu repo {0} '{1}': '{2} {3} {4} {5}', priority:{6}"
|
||||
.format(x, rep['name'], rep['type'], rep['uri'],
|
||||
rep['suite'], rep['section'], rep['priority']))
|
||||
|
||||
def report_centos_repos(self, repos):
|
||||
for x, rep in enumerate(repos):
|
||||
logger.info(
|
||||
"Centos repo {0} '{1}': '{2} {3}', priority:{4}"
|
||||
.format(x, rep['name'], rep['type'], rep['uri'],
|
||||
rep['priority']))
|
||||
|
||||
def add_ubuntu_mirrors(self, repos=None, mirrors=help_data.MIRROR_UBUNTU,
|
||||
priority=help_data.MIRROR_UBUNTU_PRIORITY):
|
||||
if not repos:
|
||||
repos = []
|
||||
# Add external Ubuntu repositories
|
||||
for x, repo_str in enumerate(mirrors.split('|')):
|
||||
repo_value = self.parse_ubuntu_repo(
|
||||
repo_str, 'ubuntu-{0}'.format(x), priority)
|
||||
if repo_value and self.check_new_ubuntu_repo(repos, repo_value):
|
||||
repos.append(repo_value)
|
||||
return repos
|
||||
|
||||
def add_centos_mirrors(self, repos=None, mirrors=help_data.MIRROR_CENTOS,
|
||||
priority=help_data.MIRROR_CENTOS_PRIORITY):
|
||||
if not repos:
|
||||
repos = []
|
||||
# Add external Centos repositories
|
||||
for x, repo_str in enumerate(mirrors.split('|')):
|
||||
repo_value = self.parse_centos_repo(repo_str, priority)
|
||||
if repo_value and self.check_new_centos_repo(repos, repo_value):
|
||||
repos.append(repo_value)
|
||||
return repos
|
||||
|
||||
def add_ubuntu_extra_mirrors(self, repos=None, prefix='extra',
|
||||
mirrors=help_data.EXTRA_DEB_REPOS,
|
||||
priority=help_data.EXTRA_DEB_REPOS_PRIORITY):
|
||||
if not repos:
|
||||
repos = []
|
||||
# Add extra Ubuntu repositories with higher priority
|
||||
for x, repo_str in enumerate(mirrors.split('|')):
|
||||
repo_value = self.parse_ubuntu_repo(
|
||||
repo_str, '{0}-{1}'.format(prefix, x), priority)
|
||||
|
||||
if repo_value and self.check_new_ubuntu_repo(repos, repo_value):
|
||||
# Remove repos that use the same name
|
||||
for repo in repos:
|
||||
if repo["name"] == repo_value["name"]:
|
||||
repos.remove(repo)
|
||||
repos.append(repo_value)
|
||||
return repos
|
||||
|
||||
def add_centos_extra_mirrors(self, repos=None,
|
||||
mirrors=help_data.EXTRA_RPM_REPOS,
|
||||
priority=help_data.EXTRA_RPM_REPOS_PRIORITY):
|
||||
if not repos:
|
||||
repos = []
|
||||
# Add extra Centos repositories
|
||||
for x, repo_str in enumerate(mirrors.split('|')):
|
||||
repo_value = self.parse_centos_repo(repo_str, priority)
|
||||
if repo_value and self.check_new_centos_repo(repos, repo_value):
|
||||
# Remove repos that use the same name
|
||||
for repo in repos:
|
||||
if repo["name"] == repo_value["name"]:
|
||||
repos.remove(repo)
|
||||
repos.append(repo_value)
|
||||
return repos
|
||||
|
||||
def check_new_ubuntu_repo(self, repos, repo_value):
|
||||
# Checks that 'repo_value' is a new unique record for Ubuntu 'repos'
|
||||
for repo in repos:
|
||||
if (repo["type"] == repo_value["type"] and
|
||||
repo["uri"] == repo_value["uri"] and
|
||||
repo["suite"] == repo_value["suite"] and
|
||||
repo["section"] == repo_value["section"]):
|
||||
return False
|
||||
return True
|
||||
|
||||
def check_new_centos_repo(self, repos, repo_value):
|
||||
# Checks that 'repo_value' is a new unique record for Centos 'repos'
|
||||
for repo in repos:
|
||||
if repo["uri"] == repo_value["uri"]:
|
||||
return False
|
||||
return True
|
||||
|
||||
def parse_ubuntu_repo(self, repo_string, name, priority):
|
||||
# Validate DEB repository string format
|
||||
results = re.search("""
|
||||
^ # [beginning of the string]
|
||||
([\w\-\.\/]+)? # group 1: optional repository name (for Nailgun)
|
||||
,? # [optional comma separator]
|
||||
(deb|deb-src) # group 2: type; search for 'deb' or 'deb-src'
|
||||
\s+ # [space separator]
|
||||
( # group 3: uri;
|
||||
\w+:\/\/ # - protocol, i.e. 'http://'
|
||||
[\w\-\.\/]+ # - hostname
|
||||
(?::\d+) # - port, i.e. ':8080', if exists
|
||||
?[\w\-\.\/]+ # - rest of the path, if exists
|
||||
) # - end of group 2
|
||||
\s+ # [space separator]
|
||||
([\w\-\.\/]+) # group 4: suite;
|
||||
\s* # [space separator], if exists
|
||||
( # group 5: section;
|
||||
[\w\-\.\/\s]* # - several space-separated names, or None
|
||||
) # - end of group 4
|
||||
,? # [optional comma separator]
|
||||
(\d+)? # group 6: optional priority of the repository
|
||||
$ # [ending of the string]""",
|
||||
repo_string.strip(), re.VERBOSE)
|
||||
if results:
|
||||
return {"name": results.group(1) or name,
|
||||
"priority": int(results.group(6) or priority),
|
||||
"type": results.group(2),
|
||||
"uri": results.group(3),
|
||||
"suite": results.group(4),
|
||||
"section": results.group(5) or ''}
|
||||
else:
|
||||
logger.error("Provided DEB repository has incorrect format: {}"
|
||||
.format(repo_string))
|
||||
|
||||
def parse_centos_repo(self, repo_string, priority):
|
||||
# Validate RPM repository string format
|
||||
results = re.search("""
|
||||
^ # [beginning of the string]
|
||||
([\w\-\.\/]+) # group 1: repo name
|
||||
, # [comma separator]
|
||||
( # group 2: uri;
|
||||
\w+:\/\/ # - protocol, i.e. 'http://'
|
||||
[\w\-\.\/]+ # - hostname
|
||||
(?::\d+) # - port, i.e. ':8080', if exists
|
||||
?[\w\-\.\/]+ # - rest of the path, if exists
|
||||
) # - end of group 2
|
||||
\s* # [space separator]
|
||||
,? # [optional comma separator]
|
||||
(\d+)? # group 3: optional priority of the repository
|
||||
$ # [ending of the string]""",
|
||||
repo_string.strip(), re.VERBOSE)
|
||||
if results:
|
||||
return {"name": results.group(1),
|
||||
"priority": int(results.group(3) or priority),
|
||||
"type": 'rpm',
|
||||
"uri": results.group(2)}
|
||||
else:
|
||||
logger.error("Provided RPM repository has incorrect format: {}"
|
||||
.format(repo_string))
|
||||
return attributes['editable']['repo_setup']['repos']
|
||||
|
||||
@download_packages_json
|
||||
@download_astute_yaml
|
||||
|
|
|
|||
|
|
@ -189,10 +189,10 @@ class RebootPlugin(TestBasic):
|
|||
# install fuel_plugin_builder on master node
|
||||
fpb.fpb_install()
|
||||
# change timeout to a new value '1'
|
||||
fpb.change_content_in_yaml(os.path.join(tasks_path, tasks_file),
|
||||
os.path.join('/tmp/', tasks_file),
|
||||
[1, 'parameters', 'timeout'],
|
||||
1)
|
||||
fpb.put_value_to_local_yaml(os.path.join(tasks_path, tasks_file),
|
||||
os.path.join('/tmp/', tasks_file),
|
||||
[1, 'parameters', 'timeout'],
|
||||
1)
|
||||
# create plugin template on the master node
|
||||
fpb.fpb_create_plugin(plugin_name)
|
||||
# replace plugin tasks with our file
|
||||
|
|
|
|||
|
|
@ -490,7 +490,9 @@ class MultiroleMultipleServices(TestBasic):
|
|||
'slave-05': ['mongo']
|
||||
}
|
||||
)
|
||||
self.fuel_web.report_repos(cluster_id)
|
||||
|
||||
repos_attr = self.get_cluster_repos(cluster_id)
|
||||
self.fuel_web.report_repos(repos_attr)
|
||||
|
||||
# (ddmitriev): No additional checks is required after deploy,
|
||||
# just make sure that all components are installed from the
|
||||
|
|
|
|||
Loading…
Reference in New Issue