Add auxiliary repositories to a just created cluster
- after uploading files to auxiliary repos, add the appropriate repository to the last created cluster - fix some issues in regenerate_repo_* scripts to allow working with completely empty repositories. Note that for patching feature the following variables should point to the local auxiliary repositories: export LOCAL_MIRROR_UBUNTU='/var/www/nailgun/ubuntu/auxiliary/' export LOCAL_MIRROR_CENTOS='/var/www/nailgun/centos/auxiliary/' The following variables added for new repos in a cluster: AUX_DEB_REPO_PRIORITY=1150 AUX_RPM_REPO_PRIORITY=15 For auxiliary repository priority on Fuel admin node hardcoded value 15, and for nailgun repo 20. - add DockerActions for performing actions with group of containers Impelements: blueprint package-fuel-components Change-Id: If3a734550659f462a3911da93c6107026ad5702a
This commit is contained in:
Dennis Dmitriev
parent
64977f3620
commit
fadb50130d
@ -153,25 +153,68 @@ def update_fuel(func):
|
||||
|
||||
remote = environment.d_env.get_admin_remote()
|
||||
|
||||
cond_upload(remote,
|
||||
settings.UPDATE_FUEL_PATH,
|
||||
settings.LOCAL_MIRROR_CENTOS,
|
||||
"(?i).*\.rpm$")
|
||||
regenerate_centos_repo(remote, settings.LOCAL_MIRROR_CENTOS)
|
||||
centos_files_count = cond_upload(
|
||||
remote, settings.UPDATE_FUEL_PATH,
|
||||
os.path.join(settings.LOCAL_MIRROR_CENTOS, 'Packages'),
|
||||
"(?i).*\.rpm$")
|
||||
|
||||
cond_upload(remote,
|
||||
settings.UPDATE_FUEL_PATH,
|
||||
settings.LOCAL_MIRROR_UBUNTU,
|
||||
"(?i).*\.deb$")
|
||||
regenerate_ubuntu_repo(remote, settings.LOCAL_MIRROR_UBUNTU)
|
||||
ubuntu_files_count = cond_upload(
|
||||
remote, settings.UPDATE_FUEL_PATH,
|
||||
os.path.join(settings.LOCAL_MIRROR_UBUNTU, 'pool/main'),
|
||||
"(?i).*\.deb$")
|
||||
|
||||
try:
|
||||
remote.execute("for container in $(dockerctl list); do"
|
||||
" dockerctl shell $container bash -c \"yum "
|
||||
"clean expire-cache;yum update -y\";dockerctl "
|
||||
"restart $container; done")
|
||||
except Exception:
|
||||
logger.exception("Fail update of Fuel's package(s).")
|
||||
cluster_id = environment.fuel_web.get_last_created_cluster()
|
||||
|
||||
if centos_files_count > 0:
|
||||
regenerate_centos_repo(remote, settings.LOCAL_MIRROR_CENTOS)
|
||||
try:
|
||||
logger.info("Updating packages in docker containers ...")
|
||||
|
||||
# FIXME: remove or replace this hardcode of repo priorities
|
||||
# when Fuel admin node will provide
|
||||
remote.execute(
|
||||
"cd /etc/yum.repos.d/;"
|
||||
"for repo in auxiliary nailgun; do"
|
||||
" sed -i '/^priority=.*/d' ./${repo}.repo;"
|
||||
"done;"
|
||||
"echo 'priority=20' >> ./nailgun.repo;"
|
||||
"echo 'priority=15' >> ./auxiliary.repo")
|
||||
|
||||
environment.docker_actions.execute_in_containers(
|
||||
cmd='yum -y install yum-plugin-priorities')
|
||||
|
||||
# Update docker containers and restart them
|
||||
|
||||
environment.docker_actions.execute_in_containers(
|
||||
cmd='yum clean expire-cache; yum update -y')
|
||||
environment.docker_actions.restart_containers()
|
||||
|
||||
except Exception:
|
||||
logger.exception("Fail update of Fuel's package(s).")
|
||||
|
||||
logger.info("Waiting for containers are started ...")
|
||||
|
||||
# Add auxiliary repository to the cluster attributes
|
||||
if settings.OPENSTACK_RELEASE_UBUNTU not in \
|
||||
settings.OPENSTACK_RELEASE:
|
||||
environment.fuel_web.add_local_centos_mirror(
|
||||
cluster_id, name="Auxiliary",
|
||||
path=settings.LOCAL_MIRROR_CENTOS,
|
||||
priority=settings.AUX_RPM_REPO_PRIORITY)
|
||||
|
||||
if ubuntu_files_count > 0:
|
||||
regenerate_ubuntu_repo(remote, settings.LOCAL_MIRROR_UBUNTU)
|
||||
# Add auxiliary repository to the cluster attributes
|
||||
if settings.OPENSTACK_RELEASE_UBUNTU in \
|
||||
settings.OPENSTACK_RELEASE:
|
||||
environment.fuel_web.add_local_ubuntu_mirror(
|
||||
cluster_id, name="Auxiliary",
|
||||
path=settings.LOCAL_MIRROR_UBUNTU,
|
||||
priority=settings.AUX_DEB_REPO_PRIORITY)
|
||||
else:
|
||||
logger.error("{0} .DEB files uploaded but won't be used"
|
||||
" because of deploying wrong release!"
|
||||
.format(ubuntu_files_count))
|
||||
return result
|
||||
return wrapper
|
||||
|
||||
|
||||
@ -83,7 +83,7 @@ class BaseActions(object):
|
||||
return (result['exit_code'] == 0)
|
||||
|
||||
def wait_for_ready_container(self, timeout=300):
|
||||
wait(lambda: self.is_container_ready, timeout)
|
||||
wait(lambda: self.is_container_ready, timeout=timeout)
|
||||
|
||||
|
||||
class AdminActions(BaseActions):
|
||||
@ -366,3 +366,35 @@ class CobblerActions(BaseActions):
|
||||
command='service dnsmasq restart',
|
||||
exit_code=0
|
||||
)
|
||||
|
||||
|
||||
class DockerActions(object):
|
||||
def __init__(self, admin_remote):
|
||||
self.admin_remote = admin_remote
|
||||
|
||||
def list_containers(self):
|
||||
return self.admin_remote.execute('dockerctl list')['stdout']
|
||||
|
||||
def wait_for_ready_containers(self, timeout=300):
|
||||
cont_actions = []
|
||||
for container in self.list_containers():
|
||||
cont_action = BaseActions(self.admin_remote)
|
||||
cont_action.container = container
|
||||
cont_actions.append(cont_action)
|
||||
wait(lambda: all([cont_action.is_container_ready
|
||||
for cont_action in cont_actions]), timeout=timeout)
|
||||
|
||||
def restart_container(self, container):
|
||||
self.admin_remote.execute('dockerctl restart {0}'.format(container))
|
||||
cont_action = BaseActions(self.admin_remote)
|
||||
cont_action.container = container
|
||||
cont_action.wait_for_ready_container()
|
||||
|
||||
def restart_containers(self):
|
||||
for container in self.list_containers():
|
||||
self.restart_container(container)
|
||||
|
||||
def execute_in_containers(self, cmd):
|
||||
for container in self.list_containers():
|
||||
self.admin_remote.execute(
|
||||
"dockerctl shell {0} bash -c '{1}'".format(container, cmd))
|
||||
|
||||
@ -2,7 +2,4 @@
|
||||
|
||||
REPO_PATH=$1
|
||||
|
||||
COMPSXML=$(awk -F'"' '$4 ~ /comps.xml$/{print $4; exit}' ${REPO_PATH}/repodata/repomd.xml)
|
||||
|
||||
createrepo -g ${REPO_PATH}/${COMPSXML} -o ${REPO_PATH} ${REPO_PATH} 2>/dev/null
|
||||
|
||||
createrepo --update ${REPO_PATH} 2>/dev/null
|
||||
|
||||
@ -298,8 +298,8 @@ class CustomRepo(object):
|
||||
assert_equal(0, script_result['exit_code'],
|
||||
self.assert_msg(script_cmd, script_result['stderr']))
|
||||
|
||||
logger.info('Local "{0}" repository {1} has been updated successfuly.'
|
||||
.format(settings.OPENSTACK_RELEASE, local_mirror_path))
|
||||
logger.info('Local repository {0} has been updated successfuly.'
|
||||
.format(local_mirror_path))
|
||||
|
||||
def assert_msg(self, cmd, err):
|
||||
return 'Executing \'{0}\' on the admin node has failed with: {1}'\
|
||||
|
||||
@ -7,14 +7,22 @@ set -e
|
||||
|
||||
ARCH=amd64
|
||||
REPO_PATH=$1
|
||||
REPONAME=$2
|
||||
SUITE=$2
|
||||
SECTION=main
|
||||
|
||||
BINDIR=${REPO_PATH}/dists/${REPONAME}/main
|
||||
release_header=`sed '/MD5Sum:/,$d' ${REPO_PATH}/dists/${REPONAME}/Release`
|
||||
BINDIR=${REPO_PATH}/dists/${SUITE}/${SECTION}
|
||||
|
||||
override_main="indices/override.${REPONAME}.main"
|
||||
override_udeb="indices/override.${REPONAME}.main.debian-installer"
|
||||
override_extra="indices/override.${REPONAME}.extra.main"
|
||||
# Validate structure of the repo
|
||||
mkdir -p "${BINDIR}/binary-${ARCH}/"
|
||||
mkdir -p "${REPO_PATH}/pool/${SECTION}"
|
||||
RELEASE="${REPO_PATH}/dists/${SUITE}/Release"
|
||||
touch ${RELEASE}
|
||||
|
||||
release_header=`sed '/MD5Sum:/,$d' ${RELEASE}`
|
||||
|
||||
override_main="indices/override.${SUITE}.${SECTION}"
|
||||
override_udeb="indices/override.${SUITE}.${SECTION}.debian-installer"
|
||||
override_extra="indices/override.${SUITE}.extra.${SECTION}"
|
||||
|
||||
if [ -f "${REPO_PATH}/${override_main}" ]; then
|
||||
binoverride="${override_main}"
|
||||
@ -38,7 +46,7 @@ package_udeb=${BINDIR}/debian-installer/binary-${ARCH}/Packages
|
||||
cd ${REPO_PATH}
|
||||
|
||||
# Scan *.deb packages
|
||||
dpkg-scanpackages -m ${extraoverride} -a ${ARCH} pool/main ${binoverride} > ${package_deb}.tmp 2>/dev/null
|
||||
dpkg-scanpackages -m ${extraoverride} -a ${ARCH} pool/${SECTION} ${binoverride} > ${package_deb}.tmp 2>/dev/null
|
||||
|
||||
gzip -9c ${package_deb}.tmp > ${package_deb}.gz.tmp
|
||||
bzip2 -ckz ${package_deb}.tmp > ${package_deb}.bz2.tmp
|
||||
@ -62,7 +70,7 @@ if [ -d "${BINDIR}/debian-installer/binary-${ARCH}/" ]; then
|
||||
fi
|
||||
|
||||
# Generate release file
|
||||
cd ${REPO_PATH}/dists/${REPONAME}
|
||||
cd ${REPO_PATH}/dists/${SUITE}
|
||||
echo "$release_header" > Release.tmp
|
||||
|
||||
# Generate hashes
|
||||
@ -72,7 +80,7 @@ c2=(md5 sha1 sha256 sha512)
|
||||
i=0
|
||||
while [ $i -lt ${#c1[*]} ]; do
|
||||
echo ${c1[i]}
|
||||
for hashme in `find main -type f \( -not -name "*~" -name "Package*" -o -name "Release*" \)`; do
|
||||
for hashme in `find ${SECTION} -type f \( -not -name "*~" -name "Package*" -o -name "Release*" \)`; do
|
||||
ohash=`openssl dgst -${c2[$i]} ${hashme}`
|
||||
chash="${ohash##* }"
|
||||
size=`stat -c %s ${hashme}`
|
||||
|
||||
@ -237,14 +237,19 @@ def cond_upload(remote, source, target, condition=''):
|
||||
if remote.isdir(target):
|
||||
target = posixpath.join(target, os.path.basename(source))
|
||||
|
||||
source = os.path.expanduser(source)
|
||||
if not os.path.isdir(source):
|
||||
if re.match(condition, source):
|
||||
remote.upload(source, target)
|
||||
return
|
||||
logger.debug("File '{0}' uploaded to the remote folder '{1}'"
|
||||
.format(source, target))
|
||||
return 1
|
||||
else:
|
||||
logger.debug("Pattern '{0}' doesn't match the file '{1}', "
|
||||
"uploading skipped".format(condition, source))
|
||||
return 0
|
||||
|
||||
files_count = 0
|
||||
for rootdir, subdirs, files in os.walk(source):
|
||||
targetdir = os.path.normpath(
|
||||
os.path.join(
|
||||
@ -258,6 +263,10 @@ def cond_upload(remote, source, target, condition=''):
|
||||
remote_path = posixpath.join(targetdir, entry)
|
||||
if re.match(condition, local_path):
|
||||
remote.upload(local_path, remote_path)
|
||||
files_count += 1
|
||||
logger.debug("File '{0}' uploaded to the remote folder '{1}'"
|
||||
.format(source, target))
|
||||
else:
|
||||
logger.debug("Pattern '{0}' doesn't match the file '{1}', "
|
||||
"uploading skipped".format(condition, local_path))
|
||||
return files_count
|
||||
|
||||
@ -30,6 +30,7 @@ from fuelweb_test.helpers.decorators import upload_manifests
|
||||
from fuelweb_test.helpers.eb_tables import Ebtables
|
||||
from fuelweb_test.helpers.fuel_actions import AdminActions
|
||||
from fuelweb_test.helpers.fuel_actions import CobblerActions
|
||||
from fuelweb_test.helpers.fuel_actions import DockerActions
|
||||
from fuelweb_test.helpers.fuel_actions import NailgunActions
|
||||
from fuelweb_test.helpers.fuel_actions import PostgresActions
|
||||
from fuelweb_test.helpers.ntp import Ntp
|
||||
@ -62,6 +63,10 @@ class EnvironmentModel(object):
|
||||
def cobbler_actions(self):
|
||||
return CobblerActions(self.d_env.get_admin_remote())
|
||||
|
||||
@property
|
||||
def docker_actions(self):
|
||||
return DockerActions(self.d_env.get_admin_remote())
|
||||
|
||||
@property
|
||||
def admin_node_ip(self):
|
||||
return self.fuel_web.admin_node_ip
|
||||
@ -305,7 +310,7 @@ class EnvironmentModel(object):
|
||||
self.set_admin_ssh_password()
|
||||
self.admin_actions.modify_configs(self.d_env.router())
|
||||
self.wait_bootstrap()
|
||||
self.nailgun_actions.wait_for_ready_container()
|
||||
self.docker_actions.wait_for_ready_containers()
|
||||
time.sleep(10)
|
||||
self.set_admin_keystone_password()
|
||||
self.sync_time()
|
||||
|
||||
@ -449,23 +449,24 @@ class FuelWebClient(object):
|
||||
|
||||
return cluster_id
|
||||
|
||||
def add_local_ubuntu_mirror(self, cluster_id,
|
||||
def add_local_ubuntu_mirror(self, cluster_id, name='Auxiliary',
|
||||
path=help_data.LOCAL_MIRROR_UBUNTU,
|
||||
suite='auxiliary', section='main',
|
||||
priority=help_data.EXTRA_DEB_REPOS_PRIORITY):
|
||||
# Append new mirror to attributes of currently creating Ubuntu cluster
|
||||
mirror_url = path.replace('/var/www/nailgun',
|
||||
'http://{0}:8080'.format(self.admin_node_ip))
|
||||
mirror = 'deb {0} {1} {2}'.format(mirror_url, suite, section)
|
||||
mirror = '{0},deb {1} {2} {3}'.format(name, mirror_url, suite, section)
|
||||
|
||||
attributes = self.client.get_cluster_attributes(cluster_id)
|
||||
|
||||
repos_attr = attributes['editable']['repo_setup']['repos']
|
||||
self.add_ubuntu_extra_mirrors(repos=repos_attr['value'], prefix=suite,
|
||||
mirrors=mirror, priority=priority)
|
||||
self.report_ubuntu_repos(repos_attr['value'])
|
||||
self.client.update_cluster_attributes(cluster_id, attributes)
|
||||
|
||||
def add_local_centos_mirror(self, cluster_id,
|
||||
def add_local_centos_mirror(self, cluster_id, name='Auxiliary',
|
||||
path=help_data.LOCAL_MIRROR_CENTOS,
|
||||
repo_name='auxiliary',
|
||||
priority=help_data.EXTRA_RPM_REPOS_PRIORITY):
|
||||
@ -479,6 +480,7 @@ class FuelWebClient(object):
|
||||
repos_attr = attributes['editable']['repo_setup']['repos']
|
||||
self.add_centos_extra_mirrors(repos=repos_attr['value'],
|
||||
mirrors=mirror, priority=priority)
|
||||
self.report_centos_repos(repos_attr['value'])
|
||||
self.client.update_cluster_attributes(cluster_id, attributes)
|
||||
|
||||
def replace_default_repos(self):
|
||||
@ -520,11 +522,7 @@ class FuelWebClient(object):
|
||||
if help_data.EXTRA_DEB_REPOS:
|
||||
self.add_ubuntu_extra_mirrors(repos=repos)
|
||||
|
||||
for x, rep in enumerate(repos):
|
||||
logger.info(
|
||||
"Ubuntu repo {0} '{1}': '{2} {3} {4} {5}', priority:{6}"
|
||||
.format(x, rep['name'], rep['type'], rep['uri'],
|
||||
rep['suite'], rep['section'], rep['priority']))
|
||||
self.report_ubuntu_repos(repos)
|
||||
return repos
|
||||
|
||||
def replace_centos_repos(self, repos_attr):
|
||||
@ -532,12 +530,23 @@ class FuelWebClient(object):
|
||||
repos = repos_attr['value']
|
||||
if help_data.EXTRA_RPM_REPOS:
|
||||
self.add_centos_extra_mirrors(repos=repos)
|
||||
|
||||
self.report_centos_repos(repos)
|
||||
return repos
|
||||
|
||||
def report_ubuntu_repos(self, repos):
|
||||
for x, rep in enumerate(repos):
|
||||
logger.info(
|
||||
"Ubuntu repo {0} '{1}': '{2} {3} {4} {5}', priority:{6}"
|
||||
.format(x, rep['name'], rep['type'], rep['uri'],
|
||||
rep['suite'], rep['section'], rep['priority']))
|
||||
|
||||
def report_centos_repos(self, repos):
|
||||
for x, rep in enumerate(repos):
|
||||
logger.info(
|
||||
"Centos repo {0} '{1}': '{2} {3}', priority:{4}"
|
||||
.format(x, rep['name'], rep['type'], rep['uri'],
|
||||
rep['priority']))
|
||||
return repos
|
||||
|
||||
def add_ubuntu_mirrors(self, repos=[], mirrors=help_data.MIRROR_UBUNTU,
|
||||
priority=help_data.MIRROR_UBUNTU_PRIORITY):
|
||||
@ -556,7 +565,12 @@ class FuelWebClient(object):
|
||||
for x, repo_str in enumerate(mirrors.split('|')):
|
||||
repo_value = self.parse_ubuntu_repo(
|
||||
repo_str, '{0}-{1}'.format(prefix, x), priority)
|
||||
|
||||
if repo_value and self.check_new_ubuntu_repo(repos, repo_value):
|
||||
# Remove repos that use the same name
|
||||
for repo in repos:
|
||||
if repo["name"] == repo_value["name"]:
|
||||
repos.remove(repo)
|
||||
repos.append(repo_value)
|
||||
return repos
|
||||
|
||||
@ -567,6 +581,10 @@ class FuelWebClient(object):
|
||||
for x, repo_str in enumerate(mirrors.split('|')):
|
||||
repo_value = self.parse_centos_repo(repo_str, priority)
|
||||
if repo_value and self.check_new_centos_repo(repos, repo_value):
|
||||
# Remove repos that use the same name
|
||||
for repo in repos:
|
||||
if repo["name"] == repo_value["name"]:
|
||||
repos.remove(repo)
|
||||
repos.append(repo_value)
|
||||
return repos
|
||||
|
||||
@ -591,29 +609,33 @@ class FuelWebClient(object):
|
||||
# Validate DEB repository string format
|
||||
results = re.search("""
|
||||
^ # [beginning of the string]
|
||||
(deb|deb-src) # group 1: type; search for 'deb' or 'deb-src'
|
||||
([\w\-\.\/]+)? # group 1: optional repository name (for Nailgun)
|
||||
,? # [optional comma separator]
|
||||
(deb|deb-src) # group 2: type; search for 'deb' or 'deb-src'
|
||||
\s+ # [space separator]
|
||||
( # group 2: uri;
|
||||
( # group 3: uri;
|
||||
\w+:\/\/ # - protocol, i.e. 'http://'
|
||||
[\w\-\.\/]+ # - hostname
|
||||
(?::\d+) # - port, i.e. ':8080', if exists
|
||||
?[\w\-\.\/]+ # - rest of the path, if exists
|
||||
) # - end of group 2
|
||||
\s+ # [space separator]
|
||||
([\w\-\.\/]+) # group 3: suite;
|
||||
([\w\-\.\/]+) # group 4: suite;
|
||||
\s* # [space separator], if exists
|
||||
( # group 4: section;
|
||||
( # group 5: section;
|
||||
[\w\-\.\/\s]* # - several space-separated names, or None
|
||||
) # - end of group 4
|
||||
,? # [optional comma separator]
|
||||
(\d+)? # group 6: optional priority of the repository
|
||||
$ # [ending of the string]""",
|
||||
repo_string.strip(), re.VERBOSE)
|
||||
if results:
|
||||
return {"name": name,
|
||||
"priority": int(priority),
|
||||
"type": results.group(1),
|
||||
"uri": results.group(2),
|
||||
"suite": results.group(3),
|
||||
"section": results.group(4) or ''}
|
||||
return {"name": results.group(1) or name,
|
||||
"priority": int(results.group(6) or priority),
|
||||
"type": results.group(2),
|
||||
"uri": results.group(3),
|
||||
"suite": results.group(4),
|
||||
"section": results.group(5) or ''}
|
||||
else:
|
||||
logger.error("Provided DEB repository has incorrect format: {}"
|
||||
.format(repo_string))
|
||||
@ -631,11 +653,13 @@ class FuelWebClient(object):
|
||||
?[\w\-\.\/]+ # - rest of the path, if exists
|
||||
) # - end of group 2
|
||||
\s* # [space separator]
|
||||
,? # [optional comma separator]
|
||||
(\d+)? # group 3: optional priority of the repository
|
||||
$ # [ending of the string]""",
|
||||
repo_string.strip(), re.VERBOSE)
|
||||
if results:
|
||||
return {"name": results.group(1),
|
||||
"priority": int(priority),
|
||||
"priority": int(results.group(3) or priority),
|
||||
"type": 'rpm',
|
||||
"uri": results.group(2)}
|
||||
else:
|
||||
|
||||
@ -391,6 +391,9 @@ MIRROR_UBUNTU_PRIORITY = os.environ.get('MIRROR_UBUNTU_PRIORITY', 1001)
|
||||
EXTRA_DEB_REPOS_PRIORITY = os.environ.get('EXTRA_DEB_REPOS_PRIORITY', 1050)
|
||||
EXTRA_RPM_REPOS = os.environ.get('EXTRA_RPM_REPOS', '')
|
||||
EXTRA_RPM_REPOS_PRIORITY = os.environ.get('EXTRA_RPM_REPOS_PRIORITY', 20)
|
||||
# Auxiliary repository priority will be set for a cluster if UPDATE_FUEL=true
|
||||
AUX_DEB_REPO_PRIORITY = os.environ.get('AUX_DEB_REPO_PRIORITY', 1150)
|
||||
AUX_RPM_REPO_PRIORITY = os.environ.get('AUX_RPM_REPO_PRIORITY', 15)
|
||||
|
||||
REPLACE_DEFAULT_REPOS = os.environ.get('REPLACE_DEFAULT_REPOS',
|
||||
'true') == 'true'
|
||||
|
||||
Loading…
Reference in New Issue
Block a user