Merge "Replace Ubuntu mirror in /etc/fuel/astute.yaml to local mirror"

This commit is contained in:
Jenkins 2015-09-10 11:59:00 +00:00 committed by Gerrit Code Review
commit 25ededc88b
6 changed files with 386 additions and 232 deletions

View File

@ -93,6 +93,11 @@ Regenerate Repo
.. automodule:: fuelweb_test.helpers.regenerate_repo .. automodule:: fuelweb_test.helpers.regenerate_repo
:members: :members:
Replace Reposiroties
--------------------
.. automodule:: fuelweb_test.helpers.replace_repos
:members:
Security Security
-------- --------
.. automodule:: fuelweb_test.helpers.security .. automodule:: fuelweb_test.helpers.security

View File

@ -29,9 +29,11 @@ from fuelweb_test.helpers import checkers
from fuelweb_test.helpers.regenerate_repo import regenerate_centos_repo from fuelweb_test.helpers.regenerate_repo import regenerate_centos_repo
from fuelweb_test.helpers.regenerate_repo import regenerate_ubuntu_repo from fuelweb_test.helpers.regenerate_repo import regenerate_ubuntu_repo
from fuelweb_test.helpers import replace_repos
from fuelweb_test.helpers.utils import cond_upload from fuelweb_test.helpers.utils import cond_upload
from fuelweb_test.settings import FUEL_PLUGIN_BUILDER_REPO from fuelweb_test.settings import FUEL_PLUGIN_BUILDER_REPO
from fuelweb_test.settings import FUEL_USE_LOCAL_NTPD from fuelweb_test.settings import FUEL_USE_LOCAL_NTPD
from fuelweb_test.settings import MIRROR_UBUNTU
from fuelweb_test import settings as hlp_data from fuelweb_test import settings as hlp_data
from fuelweb_test.settings import NESSUS_IMAGE_PATH from fuelweb_test.settings import NESSUS_IMAGE_PATH
@ -108,7 +110,7 @@ class BaseActions(object):
def wait_for_ready_container(self, timeout=300): def wait_for_ready_container(self, timeout=300):
wait(lambda: self.is_container_ready, timeout=timeout) wait(lambda: self.is_container_ready, timeout=timeout)
def change_content_in_yaml(self, old_file, new_file, element, value): def put_value_to_local_yaml(self, old_file, new_file, element, value):
"""Changes content in old_file at element is given to the new value """Changes content in old_file at element is given to the new value
and creates new file with changed content and creates new file with changed content
:param old_file: a path to the file content from to be changed :param old_file: a path to the file content from to be changed
@ -129,7 +131,32 @@ class BaseActions(object):
yaml_dict[element[-1]] = value yaml_dict[element[-1]] = value
with open(new_file, 'w') as f_new: with open(new_file, 'w') as f_new:
yaml.dump(origin_yaml, f_new) yaml.dump(origin_yaml, f_new, default_flow_style=False)
def get_value_from_local_yaml(self, yaml_file, element):
"""Get a value of the element from the local yaml file
:param str yaml_file: a path to the yaml file
:param list element:
list with path to element to be read
for example: ['root_elem', 'first_elem', 'target_elem']
if there are a few elements with equal names use integer
to identify which element should be used
:return obj: value
"""
with open(yaml_file, 'r') as f_old:
yaml_dict = yaml.load(f_old)
for i, k in enumerate(element):
try:
yaml_dict = yaml_dict[k]
except IndexError:
raise IndexError("Element {0} not found in the file {1}"
.format(element[: i + 1], f_old))
except KeyError:
raise KeyError("Element {0} not found in the file {1}"
.format(element[: i + 1], f_old))
return yaml_dict
def change_yaml_file_in_container( def change_yaml_file_in_container(
self, path_to_file, element, value, container=None): self, path_to_file, element, value, container=None):
@ -144,16 +171,69 @@ class BaseActions(object):
if not container: if not container:
container = self.container container = self.container
old_file = '/tmp/temp_file.old.yaml' old_file = '/tmp/temp_file_{0}.old.yaml'.format(str(os.getpid()))
new_file = '/tmp/temp_file.new.yaml' new_file = '/tmp/temp_file_{0}.new.yaml'.format(str(os.getpid()))
self.copy_between_node_and_container( self.copy_between_node_and_container(
'{0}:{1}'.format(container, path_to_file), old_file) '{0}:{1}'.format(container, path_to_file), old_file)
self.admin_remote.download(old_file, old_file) self.admin_remote.download(old_file, old_file)
self.change_content_in_yaml(old_file, new_file, element, value) self.put_value_to_local_yaml(old_file, new_file, element, value)
self.admin_remote.upload(new_file, new_file) self.admin_remote.upload(new_file, new_file)
self.copy_between_node_and_container( self.copy_between_node_and_container(
new_file, '{0}:{1}'.format(container, path_to_file)) new_file, '{0}:{1}'.format(container, path_to_file))
os.remove(old_file)
os.remove(new_file)
def get_value_from_yaml(self, path_to_file, element):
"""Get a value from the yaml file stored in container
or on master node if self.container is None
:param str path_to_file: absolutely path to the file
:param list element: list with path to the element be changed
:return obj: value
"""
if self.container:
admin_tmp_file = '/tmp/temp_file_{0}.yaml'.format(str(os.getpid()))
self.copy_between_node_and_container(
'{0}:{1}'.format(self.container, path_to_file), admin_tmp_file)
else:
admin_tmp_file = path_to_file
host_tmp_file = '/tmp/temp_file_{0}.yaml'.format(str(os.getpid()))
self.admin_remote.download(admin_tmp_file, host_tmp_file)
value = self.get_value_from_local_yaml(host_tmp_file, element)
os.remove(host_tmp_file)
return value
def put_value_to_yaml(self, path_to_file, element, value):
"""Put a value to the yaml file stored in container
or on master node if self.container is None
:param str path_to_file: absolutely path to the file
:param list element: list with path to the element be changed
:param value: new value for element
:return: None
"""
if self.container:
admin_tmp_file = '/tmp/temp_file_{0}.yaml'.format(str(os.getpid()))
self.copy_between_node_and_container(
'{0}:{1}'.format(self.container, path_to_file), admin_tmp_file)
else:
admin_tmp_file = path_to_file
host_tmp_file = '/tmp/temp_file_{0}.yaml'.format(str(os.getpid()))
self.admin_remote.download(admin_tmp_file, host_tmp_file)
self.put_value_to_local_yaml(host_tmp_file, host_tmp_file,
element, value)
self.admin_remote.upload(host_tmp_file, admin_tmp_file)
if self.container:
self.copy_between_node_and_container(
admin_tmp_file, '{0}:{1}'.format(self.container, path_to_file))
os.remove(host_tmp_file)
class AdminActions(BaseActions): class AdminActions(BaseActions):
@ -202,6 +282,15 @@ class AdminActions(BaseActions):
assert_equal(0, result['exit_code'], assert_equal(0, result['exit_code'],
"Command [{cmd}] failed with the following " "Command [{cmd}] failed with the following "
"result: {res}".format(cmd=cmd, res=result)) "result: {res}".format(cmd=cmd, res=result))
if MIRROR_UBUNTU:
logger.info("Replace Ubuntu mirror for bootstrap image in {0}"
.format(config))
repo_url = self.get_value_from_yaml(config, ['BOOTSTRAP',
'MIRROR_DISTRO'])
new_repo_url = replace_repos.replace_ubuntu_repo_url(
repo_url, upstream_host='archive.ubuntu.com')
self.put_value_to_yaml(config, ['BOOTSTRAP', 'MIRROR_DISTRO'],
new_repo_url)
@logwrap @logwrap
def upload_packages(self, local_packages_dir, centos_repo_path, def upload_packages(self, local_packages_dir, centos_repo_path,

View File

@ -0,0 +1,262 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import re
from fuelweb_test import logger
import fuelweb_test.settings as help_data
def replace_ubuntu_repo_url(repo_url, upstream_host):
repos_attr = {
"value": [
{
"name": "ubuntu-bootstrap",
"uri": repo_url,
"type": "deb",
"section": None,
"suite": None
}
]
}
repos = replace_ubuntu_repos(repos_attr, upstream_host)
new_repo_url = repos[0]['uri']
if new_repo_url != repo_url:
logger.debug("Ubuntu repository url changed from '{0}' to '{1}'"
.format(repo_url, new_repo_url))
return new_repo_url
def replace_ubuntu_repos(repos_attr, upstream_host):
# Walk thru repos_attr and replace/add extra Ubuntu mirrors
repos = []
if help_data.MIRROR_UBUNTU:
logger.debug("Adding new mirrors: '{0}'"
.format(help_data.MIRROR_UBUNTU))
repos = add_ubuntu_mirrors()
# Keep other (not upstream) repos, skip previously added ones
for repo_value in repos_attr['value']:
if upstream_host not in repo_value['uri']:
if check_new_ubuntu_repo(repos, repo_value):
repos.append(repo_value)
else:
logger.debug("Removing mirror: '{0} {1}'"
.format(repo_value['name'], repo_value['uri']))
else:
# Use defaults from Nailgun if MIRROR_UBUNTU is not set
repos = repos_attr['value']
if help_data.EXTRA_DEB_REPOS:
repos = add_ubuntu_extra_mirrors(repos=repos)
if help_data.PATCHING_DISABLE_UPDATES:
for repo in repos:
if repo['name'] in ('mos-updates', 'mos-security'):
repos.remove(repo)
return repos
def replace_centos_repos(repos_attr, upstream_host):
# Walk thru repos_attr and replace/add extra Centos mirrors
repos = []
if help_data.MIRROR_CENTOS:
logger.debug("Adding new mirrors: '{0}'"
.format(help_data.MIRROR_CENTOS))
repos = add_centos_mirrors()
# Keep other (not upstream) repos, skip previously added ones
for repo_value in repos_attr['value']:
# self.admin_node_ip while repo is located on master node
if upstream_host not in repo_value['uri']:
if check_new_centos_repo(repos, repo_value):
repos.append(repo_value)
else:
logger.debug("Removing mirror: '{0} {1}'"
.format(repo_value['name'], repo_value['uri']))
else:
# Use defaults from Nailgun if MIRROR_CENTOS is not set
repos = repos_attr['value']
if help_data.EXTRA_RPM_REPOS:
repos = add_centos_extra_mirrors(repos=repos)
if help_data.PATCHING_DISABLE_UPDATES:
for repo in repos:
if repo['name'] in ('mos-updates', 'mos-security'):
repos.remove(repo)
return repos
def report_repos(repos_attr, release=help_data.OPENSTACK_RELEASE):
"""Show list of reposifories for specified cluster"""
if help_data.OPENSTACK_RELEASE_UBUNTU in release:
report_ubuntu_repos(repos_attr['value'])
else:
report_centos_repos(repos_attr['value'])
def report_ubuntu_repos(repos):
for x, rep in enumerate(repos):
logger.info(
"Ubuntu repo {0} '{1}': '{2} {3} {4} {5}', priority:{6}"
.format(x, rep['name'], rep['type'], rep['uri'],
rep['suite'], rep['section'], rep['priority']))
def report_centos_repos(repos):
for x, rep in enumerate(repos):
logger.info(
"Centos repo {0} '{1}': '{2} {3}', priority:{4}"
.format(x, rep['name'], rep['type'], rep['uri'],
rep['priority']))
def add_ubuntu_mirrors(repos=None, mirrors=help_data.MIRROR_UBUNTU,
priority=help_data.MIRROR_UBUNTU_PRIORITY):
if not repos:
repos = []
# Add external Ubuntu repositories
for x, repo_str in enumerate(mirrors.split('|')):
repo_value = parse_ubuntu_repo(
repo_str, 'ubuntu-{0}'.format(x), priority)
if repo_value and check_new_ubuntu_repo(repos, repo_value):
repos.append(repo_value)
return repos
def add_centos_mirrors(repos=None, mirrors=help_data.MIRROR_CENTOS,
priority=help_data.MIRROR_CENTOS_PRIORITY):
if not repos:
repos = []
# Add external Centos repositories
for x, repo_str in enumerate(mirrors.split('|')):
repo_value = parse_centos_repo(repo_str, priority)
if repo_value and check_new_centos_repo(repos, repo_value):
repos.append(repo_value)
return repos
def add_ubuntu_extra_mirrors(repos=None, prefix='extra',
mirrors=help_data.EXTRA_DEB_REPOS,
priority=help_data.EXTRA_DEB_REPOS_PRIORITY):
if not repos:
repos = []
# Add extra Ubuntu repositories with higher priority
for x, repo_str in enumerate(mirrors.split('|')):
repo_value = parse_ubuntu_repo(
repo_str, '{0}-{1}'.format(prefix, x), priority)
if repo_value and check_new_ubuntu_repo(repos, repo_value):
# Remove repos that use the same name
for repo in repos:
if repo["name"] == repo_value["name"]:
repos.remove(repo)
repos.append(repo_value)
return repos
def add_centos_extra_mirrors(repos=None,
mirrors=help_data.EXTRA_RPM_REPOS,
priority=help_data.EXTRA_RPM_REPOS_PRIORITY):
if not repos:
repos = []
# Add extra Centos repositories
for x, repo_str in enumerate(mirrors.split('|')):
repo_value = parse_centos_repo(repo_str, priority)
if repo_value and check_new_centos_repo(repos, repo_value):
# Remove repos that use the same name
for repo in repos:
if repo["name"] == repo_value["name"]:
repos.remove(repo)
repos.append(repo_value)
return repos
def check_new_ubuntu_repo(repos, repo_value):
# Checks that 'repo_value' is a new unique record for Ubuntu 'repos'
for repo in repos:
if (repo["type"] == repo_value["type"] and
repo["uri"] == repo_value["uri"] and
repo["suite"] == repo_value["suite"] and
repo["section"] == repo_value["section"]):
return False
return True
def check_new_centos_repo(repos, repo_value):
# Checks that 'repo_value' is a new unique record for Centos 'repos'
for repo in repos:
if repo["uri"] == repo_value["uri"]:
return False
return True
def parse_ubuntu_repo(repo_string, name, priority):
# Validate DEB repository string format
results = re.search("""
^ # [beginning of the string]
([\w\-\.\/]+)? # group 1: optional repository name (for Nailgun)
,? # [optional comma separator]
(deb|deb-src) # group 2: type; search for 'deb' or 'deb-src'
\s+ # [space separator]
( # group 3: uri;
\w+:\/\/ # - protocol, i.e. 'http://'
[\w\-\.\/]+ # - hostname
(?::\d+) # - port, i.e. ':8080', if exists
?[\w\-\.\/]+ # - rest of the path, if exists
) # - end of group 2
\s+ # [space separator]
([\w\-\.\/]+) # group 4: suite;
\s* # [space separator], if exists
( # group 5: section;
[\w\-\.\/\s]* # - several space-separated names, or None
) # - end of group 4
,? # [optional comma separator]
(\d+)? # group 6: optional priority of the repository
$ # [ending of the string]""",
repo_string.strip(), re.VERBOSE)
if results:
return {"name": results.group(1) or name,
"priority": int(results.group(6) or priority),
"type": results.group(2),
"uri": results.group(3),
"suite": results.group(4),
"section": results.group(5) or ''}
else:
logger.error("Provided DEB repository has incorrect format: {}"
.format(repo_string))
def parse_centos_repo(repo_string, priority):
# Validate RPM repository string format
results = re.search("""
^ # [beginning of the string]
([\w\-\.\/]+) # group 1: repo name
, # [comma separator]
( # group 2: uri;
\w+:\/\/ # - protocol, i.e. 'http://'
[\w\-\.\/]+ # - hostname
(?::\d+) # - port, i.e. ':8080', if exists
?[\w\-\.\/]+ # - rest of the path, if exists
) # - end of group 2
\s* # [space separator]
,? # [optional comma separator]
(\d+)? # group 3: optional priority of the repository
$ # [ending of the string]""",
repo_string.strip(), re.VERBOSE)
if results:
return {"name": results.group(1),
"priority": int(results.group(3) or priority),
"type": 'rpm',
"uri": results.group(2)}
else:
logger.error("Provided RPM repository has incorrect format: {}"
.format(repo_string))

View File

@ -42,6 +42,7 @@ from fuelweb_test.helpers.decorators import retry
from fuelweb_test.helpers.decorators import update_fuel from fuelweb_test.helpers.decorators import update_fuel
from fuelweb_test.helpers.decorators import update_ostf from fuelweb_test.helpers.decorators import update_ostf
from fuelweb_test.helpers.decorators import upload_manifests from fuelweb_test.helpers.decorators import upload_manifests
from fuelweb_test.helpers import replace_repos
from fuelweb_test.helpers.security import SecurityChecks from fuelweb_test.helpers.security import SecurityChecks
from fuelweb_test.helpers.utils import run_on_remote from fuelweb_test.helpers.utils import run_on_remote
from fuelweb_test import logger from fuelweb_test import logger
@ -592,15 +593,15 @@ class FuelWebClient(object):
mirror = '{0},deb {1} {2} {3}'.format(name, mirror_url, suite, section) mirror = '{0},deb {1} {2} {3}'.format(name, mirror_url, suite, section)
attributes = self.client.get_cluster_attributes(cluster_id) attributes = self.client.get_cluster_attributes(cluster_id)
repos_attr = attributes['editable']['repo_setup']['repos'] repos_attr = attributes['editable']['repo_setup']['repos']
repos_attr['value'] = self.add_ubuntu_extra_mirrors(
repos_attr['value'] = replace_repos.add_ubuntu_extra_mirrors(
repos=repos_attr['value'], repos=repos_attr['value'],
prefix=suite, prefix=suite,
mirrors=mirror, mirrors=mirror,
priority=priority) priority=priority)
self.report_ubuntu_repos(repos_attr['value']) replace_repos.report_ubuntu_repos(repos_attr['value'])
self.client.update_cluster_attributes(cluster_id, attributes) self.client.update_cluster_attributes(cluster_id, attributes)
def add_local_centos_mirror(self, cluster_id, name='Auxiliary', def add_local_centos_mirror(self, cluster_id, name='Auxiliary',
@ -613,18 +614,19 @@ class FuelWebClient(object):
mirror = '{0},{1}'.format(repo_name, mirror_url) mirror = '{0},{1}'.format(repo_name, mirror_url)
attributes = self.client.get_cluster_attributes(cluster_id) attributes = self.client.get_cluster_attributes(cluster_id)
repos_attr = attributes['editable']['repo_setup']['repos'] repos_attr = attributes['editable']['repo_setup']['repos']
repos_attr['value'] = self.add_centos_extra_mirrors(
repos_attr['value'] = replace_repos.add_centos_extra_mirrors(
repos=repos_attr['value'], repos=repos_attr['value'],
mirrors=mirror, mirrors=mirror,
priority=priority) priority=priority)
self.report_centos_repos(repos_attr['value']) replace_repos.report_centos_repos(repos_attr['value'])
self.client.update_cluster_attributes(cluster_id, attributes) self.client.update_cluster_attributes(cluster_id, attributes)
def replace_default_repos(self): def replace_default_repos(self):
# Replace Ubuntu default repositories for the release # Replace Ubuntu default repositories for the release
logger.info("Replace default repository list.")
ubuntu_id = self.client.get_release_id( ubuntu_id = self.client.get_release_id(
release_name=help_data.OPENSTACK_RELEASE_UBUNTU) release_name=help_data.OPENSTACK_RELEASE_UBUNTU)
@ -632,8 +634,11 @@ class FuelWebClient(object):
ubuntu_meta = ubuntu_release["attributes_metadata"] ubuntu_meta = ubuntu_release["attributes_metadata"]
repos_ubuntu = ubuntu_meta["editable"]["repo_setup"]["repos"] repos_ubuntu = ubuntu_meta["editable"]["repo_setup"]["repos"]
repos_ubuntu["value"] = self.replace_ubuntu_repos(repos_ubuntu) repos_ubuntu["value"] = replace_repos.replace_ubuntu_repos(
repos_ubuntu, upstream_host='archive.ubuntu.com')
self.client.put_release(ubuntu_id, ubuntu_release) self.client.put_release(ubuntu_id, ubuntu_release)
replace_repos.report_ubuntu_repos(repos_ubuntu["value"])
# Replace CentOS default repositories for the release # Replace CentOS default repositories for the release
centos_id = self.client.get_release_id( centos_id = self.client.get_release_id(
@ -643,224 +648,15 @@ class FuelWebClient(object):
centos_meta = centos_release["attributes_metadata"] centos_meta = centos_release["attributes_metadata"]
repos_centos = centos_meta["editable"]["repo_setup"]["repos"] repos_centos = centos_meta["editable"]["repo_setup"]["repos"]
repos_centos["value"] = self.replace_centos_repos(repos_centos) repos_centos["value"] = replace_repos.replace_centos_repos(
repos_centos, upstream_host=self.admin_node_ip)
self.client.put_release(centos_id, centos_release) self.client.put_release(centos_id, centos_release)
replace_repos.report_centos_repos(repos_centos["value"])
def replace_ubuntu_repos(self, repos_attr): def get_cluster_repos(self, cluster_id):
# Walk thru repos_attr and replace/add extra Ubuntu mirrors
repos = []
if help_data.MIRROR_UBUNTU:
logger.info("Adding new mirrors: '{0}'"
.format(help_data.MIRROR_UBUNTU))
repos = self.add_ubuntu_mirrors()
# Keep other (not upstream) repos, skip previously added ones
for repo_value in repos_attr['value']:
if 'archive.ubuntu.com' not in repo_value['uri']:
if self.check_new_ubuntu_repo(repos, repo_value):
repos.append(repo_value)
else:
logger.warning("Removing mirror: '{0}'"
.format(repo_value['uri']))
else:
# Use defaults from Nailgun if MIRROR_UBUNTU is not set
repos = repos_attr['value']
if help_data.EXTRA_DEB_REPOS:
repos = self.add_ubuntu_extra_mirrors(repos=repos)
if help_data.PATCHING_DISABLE_UPDATES:
for repo in repos:
if repo['name'] in ('mos-updates', 'mos-security'):
repos.remove(repo)
self.report_ubuntu_repos(repos)
return repos
def replace_centos_repos(self, repos_attr):
# Walk thru repos_attr and replace/add extra Centos mirrors
repos = []
if help_data.MIRROR_CENTOS:
logger.info("Adding new mirrors: '{0}'"
.format(help_data.MIRROR_CENTOS))
repos = self.add_centos_mirrors()
# Keep other (not upstream) repos, skip previously added ones
for repo_value in repos_attr['value']:
# self.admin_node_ip while repo is located on master node
if self.admin_node_ip not in repo_value['uri']:
if self.check_new_centos_repo(repos, repo_value):
repos.append(repo_value)
else:
logger.warning("Removing mirror: '{0}'"
.format(repo_value['uri']))
else:
# Use defaults from Nailgun if MIRROR_CENTOS is not set
repos = repos_attr['value']
if help_data.EXTRA_RPM_REPOS:
repos = self.add_centos_extra_mirrors(repos=repos)
if help_data.PATCHING_DISABLE_UPDATES:
for repo in repos:
if repo['name'] in ('mos-updates', 'mos-security'):
repos.remove(repo)
self.report_centos_repos(repos)
return repos
def report_repos(self, cluster_id, release=help_data.OPENSTACK_RELEASE):
"""Show list of reposifories for specified cluster"""
attributes = self.client.get_cluster_attributes(cluster_id) attributes = self.client.get_cluster_attributes(cluster_id)
repos_attr = attributes['editable']['repo_setup']['repos'] return attributes['editable']['repo_setup']['repos']
if help_data.OPENSTACK_RELEASE_UBUNTU in release:
self.report_ubuntu_repos(repos_attr['value'])
else:
self.report_centos_repos(repos_attr['value'])
def report_ubuntu_repos(self, repos):
for x, rep in enumerate(repos):
logger.info(
"Ubuntu repo {0} '{1}': '{2} {3} {4} {5}', priority:{6}"
.format(x, rep['name'], rep['type'], rep['uri'],
rep['suite'], rep['section'], rep['priority']))
def report_centos_repos(self, repos):
for x, rep in enumerate(repos):
logger.info(
"Centos repo {0} '{1}': '{2} {3}', priority:{4}"
.format(x, rep['name'], rep['type'], rep['uri'],
rep['priority']))
def add_ubuntu_mirrors(self, repos=None, mirrors=help_data.MIRROR_UBUNTU,
priority=help_data.MIRROR_UBUNTU_PRIORITY):
if not repos:
repos = []
# Add external Ubuntu repositories
for x, repo_str in enumerate(mirrors.split('|')):
repo_value = self.parse_ubuntu_repo(
repo_str, 'ubuntu-{0}'.format(x), priority)
if repo_value and self.check_new_ubuntu_repo(repos, repo_value):
repos.append(repo_value)
return repos
def add_centos_mirrors(self, repos=None, mirrors=help_data.MIRROR_CENTOS,
priority=help_data.MIRROR_CENTOS_PRIORITY):
if not repos:
repos = []
# Add external Centos repositories
for x, repo_str in enumerate(mirrors.split('|')):
repo_value = self.parse_centos_repo(repo_str, priority)
if repo_value and self.check_new_centos_repo(repos, repo_value):
repos.append(repo_value)
return repos
def add_ubuntu_extra_mirrors(self, repos=None, prefix='extra',
mirrors=help_data.EXTRA_DEB_REPOS,
priority=help_data.EXTRA_DEB_REPOS_PRIORITY):
if not repos:
repos = []
# Add extra Ubuntu repositories with higher priority
for x, repo_str in enumerate(mirrors.split('|')):
repo_value = self.parse_ubuntu_repo(
repo_str, '{0}-{1}'.format(prefix, x), priority)
if repo_value and self.check_new_ubuntu_repo(repos, repo_value):
# Remove repos that use the same name
for repo in repos:
if repo["name"] == repo_value["name"]:
repos.remove(repo)
repos.append(repo_value)
return repos
def add_centos_extra_mirrors(self, repos=None,
mirrors=help_data.EXTRA_RPM_REPOS,
priority=help_data.EXTRA_RPM_REPOS_PRIORITY):
if not repos:
repos = []
# Add extra Centos repositories
for x, repo_str in enumerate(mirrors.split('|')):
repo_value = self.parse_centos_repo(repo_str, priority)
if repo_value and self.check_new_centos_repo(repos, repo_value):
# Remove repos that use the same name
for repo in repos:
if repo["name"] == repo_value["name"]:
repos.remove(repo)
repos.append(repo_value)
return repos
def check_new_ubuntu_repo(self, repos, repo_value):
# Checks that 'repo_value' is a new unique record for Ubuntu 'repos'
for repo in repos:
if (repo["type"] == repo_value["type"] and
repo["uri"] == repo_value["uri"] and
repo["suite"] == repo_value["suite"] and
repo["section"] == repo_value["section"]):
return False
return True
def check_new_centos_repo(self, repos, repo_value):
# Checks that 'repo_value' is a new unique record for Centos 'repos'
for repo in repos:
if repo["uri"] == repo_value["uri"]:
return False
return True
def parse_ubuntu_repo(self, repo_string, name, priority):
# Validate DEB repository string format
results = re.search("""
^ # [beginning of the string]
([\w\-\.\/]+)? # group 1: optional repository name (for Nailgun)
,? # [optional comma separator]
(deb|deb-src) # group 2: type; search for 'deb' or 'deb-src'
\s+ # [space separator]
( # group 3: uri;
\w+:\/\/ # - protocol, i.e. 'http://'
[\w\-\.\/]+ # - hostname
(?::\d+) # - port, i.e. ':8080', if exists
?[\w\-\.\/]+ # - rest of the path, if exists
) # - end of group 2
\s+ # [space separator]
([\w\-\.\/]+) # group 4: suite;
\s* # [space separator], if exists
( # group 5: section;
[\w\-\.\/\s]* # - several space-separated names, or None
) # - end of group 4
,? # [optional comma separator]
(\d+)? # group 6: optional priority of the repository
$ # [ending of the string]""",
repo_string.strip(), re.VERBOSE)
if results:
return {"name": results.group(1) or name,
"priority": int(results.group(6) or priority),
"type": results.group(2),
"uri": results.group(3),
"suite": results.group(4),
"section": results.group(5) or ''}
else:
logger.error("Provided DEB repository has incorrect format: {}"
.format(repo_string))
def parse_centos_repo(self, repo_string, priority):
# Validate RPM repository string format
results = re.search("""
^ # [beginning of the string]
([\w\-\.\/]+) # group 1: repo name
, # [comma separator]
( # group 2: uri;
\w+:\/\/ # - protocol, i.e. 'http://'
[\w\-\.\/]+ # - hostname
(?::\d+) # - port, i.e. ':8080', if exists
?[\w\-\.\/]+ # - rest of the path, if exists
) # - end of group 2
\s* # [space separator]
,? # [optional comma separator]
(\d+)? # group 3: optional priority of the repository
$ # [ending of the string]""",
repo_string.strip(), re.VERBOSE)
if results:
return {"name": results.group(1),
"priority": int(results.group(3) or priority),
"type": 'rpm',
"uri": results.group(2)}
else:
logger.error("Provided RPM repository has incorrect format: {}"
.format(repo_string))
@download_packages_json @download_packages_json
@download_astute_yaml @download_astute_yaml

View File

@ -189,10 +189,10 @@ class RebootPlugin(TestBasic):
# install fuel_plugin_builder on master node # install fuel_plugin_builder on master node
fpb.fpb_install() fpb.fpb_install()
# change timeout to a new value '1' # change timeout to a new value '1'
fpb.change_content_in_yaml(os.path.join(tasks_path, tasks_file), fpb.put_value_to_local_yaml(os.path.join(tasks_path, tasks_file),
os.path.join('/tmp/', tasks_file), os.path.join('/tmp/', tasks_file),
[1, 'parameters', 'timeout'], [1, 'parameters', 'timeout'],
1) 1)
# create plugin template on the master node # create plugin template on the master node
fpb.fpb_create_plugin(plugin_name) fpb.fpb_create_plugin(plugin_name)
# replace plugin tasks with our file # replace plugin tasks with our file

View File

@ -521,7 +521,9 @@ class MultiroleMultipleServices(TestBasic):
'slave-05': ['mongo'] 'slave-05': ['mongo']
} }
) )
self.fuel_web.report_repos(cluster_id)
repos_attr = self.get_cluster_repos(cluster_id)
self.fuel_web.report_repos(repos_attr)
# (ddmitriev): No additional checks is required after deploy, # (ddmitriev): No additional checks is required after deploy,
# just make sure that all components are installed from the # just make sure that all components are installed from the