From b42978f1bdf2b4b81325bee9669a38b7f1d78743 Mon Sep 17 00:00:00 2001 From: Martin Magr Date: Fri, 4 Oct 2013 10:31:00 +0200 Subject: [PATCH] Improves EPEL and RDO repo setup This patch improves repo setup. In case of EPEL it also enables/disables EPEL repo according to configuration. In case of RDO it's installed on each host if the repo is installed locally. Fixes: rhbz#974971, rhbz#990642 Change-Id: I3a1f77061439f1072a0932918384b939babcd3e0 --- packstack/installer/run_setup.py | 2 +- packstack/installer/setup_controller.py | 12 +- packstack/modules/common.py | 7 +- .../{postscript_949.py => postscript_948.py} | 0 packstack/plugins/prescript_000.py | 17 ++ packstack/plugins/puppet_950.py | 6 +- .../{serverprep_901.py => serverprep_949.py} | 155 ++++++++++++------ tests/installer/test_run_setup.py | 10 +- tests/test_plugin_serverprep.py | 40 ++--- 9 files changed, 165 insertions(+), 84 deletions(-) rename packstack/plugins/{postscript_949.py => postscript_948.py} (100%) rename packstack/plugins/{serverprep_901.py => serverprep_949.py} (80%) diff --git a/packstack/installer/run_setup.py b/packstack/installer/run_setup.py index c4829a6d9..12ae03e3b 100644 --- a/packstack/installer/run_setup.py +++ b/packstack/installer/run_setup.py @@ -587,7 +587,7 @@ def remove_remote_var_dirs(): """ for host in gethostlist(controller.CONF): try: - host_dir = controller.temp_map[host] + host_dir = controller.CONF['HOST_DETAILS'][host]['tmpdir'] except KeyError: # Nothing was added to this host yet, so we have nothing to delete continue diff --git a/packstack/installer/setup_controller.py b/packstack/installer/setup_controller.py index c97e9a9a2..e9ffcb6c5 100644 --- a/packstack/installer/setup_controller.py +++ b/packstack/installer/setup_controller.py @@ -36,17 +36,11 @@ class Controller(object): return self.__single def __init__(self): - # XXX: Right now this will only hold all temp dirs on each host. - # Method for temp dir creation should be implemented in this - # class, when it will start behaving like controller and not - # only like data container - self.temp_map = {} - # Resources that should be copied to each host along with the puppet # files, on the remote host the file will be placed in - # $PACKSTACK_VAR_DIR/resources. As with temp_map, this controller - # should copy the files, for now the puppet plugin is doing it - # format {'host':[('/path/to/fileordirectory', 'filenameonremotehost'), ..]} + # $PACKSTACK_VAR_DIR/resources. This controller should copy the files, + # for now the puppet plugin is doing it format + # {'host':[('/path/to/fileordirectory', 'filenameonremotehost'), ..]} self.resources = {} diff --git a/packstack/modules/common.py b/packstack/modules/common.py index 9aae0727b..03f6722ed 100644 --- a/packstack/modules/common.py +++ b/packstack/modules/common.py @@ -15,6 +15,8 @@ def filtered_hosts(config, exclude=True, dbhost=True): result = set() dbinst = config.get('CONFIG_MYSQL_INSTALL') == 'y' for hosttype, hostname in utils.host_iter(config): + # if dbhost is being taken into account and we are not installing MySQL + # then we should omit the MySQL host if dbhost and not dbinst and hosttype == 'CONFIG_MYSQL_HOST': continue result.add(hostname) @@ -28,4 +30,7 @@ def is_all_in_one(config): Returns True if packstack is running allinone setup, otherwise returns False. """ - return len(filtered_hosts(config, exclude=False)) == 1 + # Even if some host have been excluded from installation, we must count + # with them when checking all-in-one. MySQL host should however be omitted + # if we are not installing MySQL + return len(filtered_hosts(config, exclude=False, dbhost=True)) == 1 diff --git a/packstack/plugins/postscript_949.py b/packstack/plugins/postscript_948.py similarity index 100% rename from packstack/plugins/postscript_949.py rename to packstack/plugins/postscript_948.py diff --git a/packstack/plugins/prescript_000.py b/packstack/plugins/prescript_000.py index 96d36afe1..db12a35ed 100644 --- a/packstack/plugins/prescript_000.py +++ b/packstack/plugins/prescript_000.py @@ -279,6 +279,11 @@ def disable_nm(config): def discover(config): + """ + Discovers details about hosts. + """ + # TODO: Once Controller is refactored, move this function to it (facter can + # be used for that too). details = {} release_regexp = re.compile(r'^(?P.*) release (?P[\d\.]*)') for host in filtered_hosts(config): @@ -303,6 +308,18 @@ def discover(config): opsys = re.sub(pattern, surr, opsys) details[host]['os'] = opsys details[host]['release'] = match.group('release') + + # Create the packstack tmp directory + server.clear() + server.append("mkdir -p %s" % basedefs.PACKSTACK_VAR_DIR) + # Separately create the tmp directory for this packstack run, this will + # fail if the directory already exists + host_dir = os.path.join(basedefs.PACKSTACK_VAR_DIR, uuid.uuid4().hex) + server.append("mkdir --mode 0700 %s" % host_dir) + for i in ('modules', 'resources'): + server.append("mkdir --mode 0700 %s" % os.path.join(host_dir, i)) + server.execute() + details[host]['tmpdir'] = host_dir config['HOST_DETAILS'] = details diff --git a/packstack/plugins/puppet_950.py b/packstack/plugins/puppet_950.py index a7e057117..f622545cc 100644 --- a/packstack/plugins/puppet_950.py +++ b/packstack/plugins/puppet_950.py @@ -92,7 +92,7 @@ def copyPuppetModules(config): server = utils.ScriptRunner() for hostname in filtered_hosts(config): - host_dir = controller.temp_map[hostname] + host_dir = config['HOST_DETAILS'][hostname]['tmpdir'] server.append("cd %s/puppet" % basedefs.DIR_PROJECT_DIR) # copy Packstack facts server.append("tar --dereference -cpzf - facts | " @@ -186,11 +186,11 @@ def applyPuppetManifest(config): if "%s_" % hostname not in manifest: continue - host_dir = controller.temp_map[hostname] + host_dir = config['HOST_DETAILS'][hostname]['tmpdir'] print "Applying " + manifest server = utils.ScriptRunner(hostname) - man_path = os.path.join(controller.temp_map[hostname], + man_path = os.path.join(config['HOST_DETAILS'][hostname]['tmpdir'], basedefs.PUPPET_MANIFEST_RELATIVE, manifest) diff --git a/packstack/plugins/serverprep_901.py b/packstack/plugins/serverprep_949.py similarity index 80% rename from packstack/plugins/serverprep_901.py rename to packstack/plugins/serverprep_949.py index 2d724553e..ad65c0ce7 100644 --- a/packstack/plugins/serverprep_901.py +++ b/packstack/plugins/serverprep_949.py @@ -3,6 +3,7 @@ prepare server """ import os +import re import uuid import logging import platform @@ -12,7 +13,7 @@ from packstack.installer import exceptions from packstack.installer import utils from packstack.installer import validators -from packstack.modules.common import filtered_hosts +from packstack.modules.common import filtered_hosts, is_all_in_one # Controller object will be initialized from main flow controller = None @@ -242,7 +243,8 @@ def initConfig(controllerObject): "POST_CONDITION_MATCH" : True}, ] - if is_rhel(): + if ((is_all_in_one(controller.CONF) and is_rhel()) or + not is_all_in_one(controller.CONF)): conf_groups.append({"GROUP_NAME" : "RHEL", "DESCRIPTION" : "RHEL config", "PRE_CONDITION" : lambda x: 'yes', @@ -365,6 +367,87 @@ def run_rhsm_reg(host, username, password, beta): server.execute(maskList=[password]) +def manage_epel(host, config): + """ + Installs and/or enables EPEL repo if it is required or disables it if it + is not required. + """ + mirrors = ('https://mirrors.fedoraproject.org/metalink?repo=epel-6&' + 'arch=$basearch') + server = utils.ScriptRunner(host) + if (config['CONFIG_USE_EPEL'] == 'y' and + config['HOST_DETAILS'][host]['os'] != 'Fedora'): + server.append('REPOFILE=$(mktemp)') + server.append('cat /etc/yum.conf > $REPOFILE') + server.append("echo -e '[packstack-epel]\nname=packstack-epel\n" + "enabled=1\nmirrorlist=%(mirrors)s' >> $REPOFILE" + % locals()) + server.append('( rpm -q epel-release ||' + ' yum install -y --nogpg -c $REPOFILE epel-release ) ' + '|| true') + server.append('rm -rf $REPOFILE') + try: + server.execute() + except exceptions.ScriptRuntimeError as ex: + msg = 'Failed to set EPEL repo on host %s:\n%s' % (host, ex) + raise exceptions.ScriptRuntimeError(msg) + + if config['CONFIG_USE_EPEL'] == 'y': + cmd = 'enable' + enabled = '(1|True)' + else: + cmd = 'disable' + enabled = '(0|False)' + server.clear() + server.append('yum-config-manager --%(cmd)s epel' % locals()) + # yum-config-manager returns 0 always, but returns current setup if succeeds + rc, out = server.execute() + match = re.search('enabled\s*\=\s*%(enabled)s' % locals(), out) + if not match: + msg = ('Failed to set EPEL repo on host %s:\nRPM file seems to be ' + 'installed, but appropriate repo file is probably missing ' + 'in /etc/yum.repos.d/' % host) + raise exceptions.ScriptRuntimeError(msg) + + +def manage_rdo(host, config): + """ + Installs and enables RDO repo on host in case it is installed locally. + """ + try: + cmd = "rpm -q rdo-release --qf='%{version}-%{release}.%{arch}\n'" + rc, out = utils.execute(cmd, use_shell=True) + except exceptions.ExecuteRuntimeError: + # RDO repo is not installed, so we don't need to continue + return + match = re.match(r'^(?P\w+)\-(?P\d+\.[\d\w]+)\n', out) + version, release = match.group('version'), match.group('release') + rdo_url = ("http://rdo.fedorapeople.org/openstack/openstack-%(version)s/" + "rdo-release-%(version)s-%(release)s.rpm" % locals()) + + server = utils.ScriptRunner(host) + server.append("(rpm -q 'rdo-release-%(version)s' ||" + " yum install -y --nogpg %(rdo_url)s) || true" + % locals()) + try: + server.execute() + except exceptions.ScriptRuntimeError as ex: + msg = 'Failed to set RDO repo on host %s:\n%s' % (host, ex) + raise exceptions.ScriptRuntimeError(msg) + + reponame = 'openstack-%s' % version + server.clear() + server.append('yum-config-manager --enable %(reponame)s' % locals()) + # yum-config-manager returns 0 always, but returns current setup if succeeds + rc, out = server.execute() + match = re.search('enabled\s*=\s*(1|True)', out) + if not match: + msg = ('Failed to set RDO repo on host %s:\nRPM file seems to be ' + 'installed, but appropriate repo file is probably missing ' + 'in /etc/yum.repos.d/' % host) + raise exceptions.ScriptRuntimeError(msg) + + def initSequences(controller): preparesteps = [ {'title': 'Preparing servers', 'functions':[serverprep]} @@ -400,7 +483,8 @@ def serverprep(config): for hostname in filtered_hosts(config): # Subscribe to Red Hat Repositories if configured if rh_username: - run_rhsm_reg(hostname, rh_username, rh_password, config["CONFIG_RH_BETA_REPO"] == 'y') + run_rhsm_reg(hostname, rh_username, rh_password, + config["CONFIG_RH_BETA_REPO"] == 'y') # Subscribe to RHN Satellite if configured if sat_url and hostname not in sat_registered: @@ -408,58 +492,33 @@ def serverprep(config): sat_registered.add(hostname) server = utils.ScriptRunner(hostname) + server.append('rpm -q --whatprovides yum-utils || ' + 'yum install -y yum-utils') + server.execute() - # install epel if on rhel (or popular derivative thereof) and epel is configured - if config["CONFIG_USE_EPEL"] == 'y': - server.append("REPOFILE=$(mktemp)") - server.append("cat /etc/yum.conf > $REPOFILE") - server.append("echo -e '[packstack-epel]\nname=packstack-epel\n" - "enabled=1\n" - "mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=epel-6&arch=$basearch'" - ">> $REPOFILE") + # enable or disable EPEL according to configuration + manage_epel(hostname, config) + # enable RDO if it is installed locally + manage_rdo(hostname, config) - server.append("grep -e 'Red Hat Enterprise Linux' -e 'CentOS' -e 'Scientific Linux' /etc/redhat-release && " - "( rpm -q epel-release || yum install -y --nogpg -c $REPOFILE epel-release ) || echo -n ''") - server.append("rm -rf $REPOFILE") - - - # set highest priority of RHOS repository if EPEL is installed and - # the repo rhel-server-ost-6-folsom-rpms exists in redhat.repo - - # If RHOS has been installed we can diable EPEL when installing openstack-utils - yum_opts = "" - if rh_username: - yum_opts += "--disablerepo='epel*'" - - server.append("rpm -q epel-release && " - "yum install -y %s openstack-utils yum-plugin-priorities || true" % yum_opts) - subs_cmd = ('rpm -q epel-release && ' - 'grep %(repo)s %(repo_file)s && ' - 'openstack-config --set %(repo_file)s %(repo)s priority %(priority)s || true') - server.append(subs_cmd % {"repo_file": "/etc/yum.repos.d/redhat.repo", - "repo": "rhel-server-ost-6-folsom-rpms", - "priority": 1}) - - # Create the packstack tmp directory - if hostname not in controller.temp_map: - # TO-DO: Move this to packstack.installer.setup_controller - server.append("mkdir -p %s" % basedefs.PACKSTACK_VAR_DIR) - # Separately create the tmp directory for this packstack run, this will fail if - # the directory already exists - host_dir = os.path.join(basedefs.PACKSTACK_VAR_DIR, uuid.uuid4().hex) - server.append("mkdir --mode 0700 %s" % host_dir) - server.append("mkdir %s/resources" % host_dir) - server.append("mkdir --mode 0700 %s" % - os.path.join(host_dir, 'modules')) - controller.temp_map[hostname] = host_dir + reponame = 'rhel-server-ost-6-4-rpms' + server.clear() + server.append('yum install -y yum-plugin-priorities || true') + # TO-DO: enable this once we will have RHN channel for Havana + #server.append('rpm -q epel-release && yum-config-manager ' + # '--setopt="%(reponame)s.priority=1" ' + # '--save %(reponame)s' % locals()) # Add yum repositories if configured CONFIG_REPO = config["CONFIG_REPO"].strip() if CONFIG_REPO: - for i, url in enumerate(CONFIG_REPO.split(',')): + for i, repourl in enumerate(CONFIG_REPO.split(',')): reponame = 'packstack_%d' % i - server.append('echo "[%s]\nname=%s\nbaseurl=%s\nenabled=1\npriority=1\ngpgcheck=0"' - ' > /etc/yum.repos.d/%s.repo' % (reponame, reponame, url, reponame)) + server.append('echo "[%(reponame)s]\nname=%(reponame)s\n' + 'baseurl=%(repourl)s\nenabled=1\n' + 'priority=1\ngpgcheck=0"' + ' > /etc/yum.repos.d/%(reponame)s.repo' + % locals()) server.append("yum clean metadata") server.execute() diff --git a/tests/installer/test_run_setup.py b/tests/installer/test_run_setup.py index 80fcc9b71..3bab8c7ac 100644 --- a/tests/installer/test_run_setup.py +++ b/tests/installer/test_run_setup.py @@ -17,13 +17,14 @@ import os import shutil +import subprocess import sys from unittest import TestCase from packstack.modules import ospluginutils, puppet from packstack.installer import run_setup, basedefs -from ..test_base import PackstackTestCaseMixin +from ..test_base import PackstackTestCaseMixin, FakePopen class CommandLineTestCase(PackstackTestCaseMixin, TestCase): @@ -42,6 +43,11 @@ class CommandLineTestCase(PackstackTestCaseMixin, TestCase): Popen is replaced in PackstackTestCaseMixin so no actual commands get run on the host running the unit tests """ + # we need following to pass manage_epel(enabled=1) and + # manage_rdo(havana-6.noarch\nenabled=0) functions + fake = FakePopen() + fake.stdout = 'havana-6.noarch\nenabled=0enabled=1' + subprocess.Popen = fake # create a dummy public key dummy_public_key = os.path.join(self.tempdir, 'id_rsa.pub') @@ -51,7 +57,7 @@ class CommandLineTestCase(PackstackTestCaseMixin, TestCase): orig_argv = sys.argv sys.argv = ['packstack', '--ssh-public-key=%s' % dummy_public_key, '--install-hosts=127.0.0.1', '--os-swift-install=y', - '--nagios-install=y'] + '--nagios-install=y', '--use-epel=y'] # There is no puppet logfile to validate, so replace # ospluginutils.validate_puppet_logfile with a mock function diff --git a/tests/test_plugin_serverprep.py b/tests/test_plugin_serverprep.py index 479eda27b..92c93e4b6 100644 --- a/tests/test_plugin_serverprep.py +++ b/tests/test_plugin_serverprep.py @@ -18,10 +18,10 @@ import os from unittest import TestCase from test_base import PackstackTestCaseMixin -from packstack.plugins import serverprep_901 +from packstack.plugins import serverprep_949 from packstack.installer.setup_controller import Controller -serverprep_901.controller = Controller() +serverprep_949.controller = Controller() class OSPluginUtilsTestCase(PackstackTestCaseMixin, TestCase): @@ -30,30 +30,30 @@ class OSPluginUtilsTestCase(PackstackTestCaseMixin, TestCase): # On non-RHEL, the CONFIG_{RH,SATELLITE} options are never set, # i.e. this test would always fail. Therefore, only run it on RHEL. - if not serverprep_901.is_rhel(): + if not serverprep_949.is_rhel(): return password = "dasd|'asda%>