From 2548b37a1bb17297305a81f832a6dfffb66fb8fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20M=C3=A1gr?= Date: Wed, 1 Apr 2015 16:20:57 +0200 Subject: [PATCH] Improve discovery logic This patch improves hosts' discovery logic by using Facter to do the work. Original code was taken from Kanzo (https://github.com/paramite/kanzo) Prerequisite for fixing rhbz#1200604 Change-Id: I8de024e55679d70a5d16f48df51337ebfea37514 --- packstack/installer/basedefs.py | 3 + packstack/installer/run_setup.py | 4 +- packstack/plugins/prescript_000.py | 106 +++++++++++++++++----------- packstack/plugins/puppet_950.py | 44 ------------ packstack/plugins/serverprep_001.py | 7 +- tests/installer/test_run_setup.py | 5 ++ 6 files changed, 80 insertions(+), 89 deletions(-) diff --git a/packstack/installer/basedefs.py b/packstack/installer/basedefs.py index 62cfa12b3..bab832a4c 100644 --- a/packstack/installer/basedefs.py +++ b/packstack/installer/basedefs.py @@ -67,6 +67,9 @@ PUPPET_MANIFEST_DIR = os.path.join(VAR_DIR, PUPPET_MANIFEST_RELATIVE) HIERADATA_FILE_RELATIVE = "hieradata" HIERADATA_DIR = os.path.join(VAR_DIR, HIERADATA_FILE_RELATIVE) +PUPPET_DEPENDENCIES = ['puppet', 'hiera', 'openssh-clients', 'tar', 'nc'] +PUPPET_MODULES_PKG = 'openstack-puppet-modules' + FILE_INSTALLER_LOG = "setup.log" DIR_PROJECT_DIR = os.environ.get('INSTALLER_PROJECT_DIR', os.path.join(os.getcwd(), 'packstack')) diff --git a/packstack/installer/run_setup.py b/packstack/installer/run_setup.py index 17ba04dc3..cb373d0ab 100644 --- a/packstack/installer/run_setup.py +++ b/packstack/installer/run_setup.py @@ -36,7 +36,7 @@ from .exceptions import FlagValidationError from .exceptions import ParamValidationError from packstack import version -from packstack.modules.ospluginutils import gethostlist +from packstack.modules.common import filtered_hosts from setup_controller import Controller controller = Controller() @@ -667,7 +667,7 @@ def remove_remote_var_dirs(options, config, messages): Removes the temp directories on remote hosts, doesn't remove data on localhost """ - for host in gethostlist(config): + for host in filtered_hosts(config): try: host_dir = config['HOST_DETAILS'][host]['tmpdir'] except KeyError: diff --git a/packstack/plugins/prescript_000.py b/packstack/plugins/prescript_000.py index 7f602d368..bf2afed68 100644 --- a/packstack/plugins/prescript_000.py +++ b/packstack/plugins/prescript_000.py @@ -18,11 +18,9 @@ Plugin responsible for setting OpenStack global options import glob import os -import re import uuid from packstack.installer import basedefs -from packstack.installer import exceptions from packstack.installer import processors from packstack.installer import utils from packstack.installer import validators @@ -552,8 +550,8 @@ def initSequences(controller): prescript_steps = [ {'title': 'Setting up ssh keys', 'functions': [install_keys]}, - {'title': 'Discovering hosts\' details', - 'functions': [discover]}, + {'title': 'Preinstalling Puppet and discovering hosts\' details', + 'functions': [preinstall_and_discover]}, {'title': 'Adding pre install manifest entries', 'functions': [create_manifest]}, ] @@ -602,49 +600,76 @@ def install_keys(config, messages): install_keys_on_host(hostname, sshkeydata) -def discover(config, messages): +def preinstall_and_discover(config, messages): + """Installs Puppet and it's dependencies and dependencies of Puppet + modules' package and discovers information about all hosts. """ - Discovers details about hosts. - """ - # TODO: Once Controller is refactored, move this function to it (facter can - # be used for that too). - details = {} - release_regexp = re.compile(r'^(?P.*) release (?P[\d\.]*)') config['HOST_LIST'] = list(filtered_hosts(config)) - for host in config['HOST_LIST']: - details.setdefault(host, {}) - server = utils.ScriptRunner(host) - # discover OS and release - server.append('cat /etc/redhat-release') - try: - rc, out = server.execute() - match = release_regexp.search(out) - if not match: - raise exceptions.ScriptRuntimeError() - except exceptions.ScriptRuntimeError: - details[host]['os'] = 'Unknown' - details[host]['release'] = 'Unknown' - else: - opsys = match.group('OS') - for pattern, surr in [('^Red Hat Enterprise Linux.*', 'RHEL'), - ('^Fedora.*', 'Fedora'), - ('^CentOS.*', 'CentOS'), - ('^Scientific Linux.*', 'SL')]: - opsys = re.sub(pattern, surr, opsys) - details[host]['os'] = opsys - details[host]['release'] = match.group('release') - # Create the packstack tmp directory + local = utils.ScriptRunner() + local.append('rpm -q --requires %s | egrep -v "^(rpmlib|\/|perl)"' + % basedefs.PUPPET_MODULES_PKG) + # this can fail if there are no dependencies other than those + # filtered out by the egrep expression. + rc, modules_deps = local.execute(can_fail=False) + + # modules package might not be installed if we are running from source; + # in this case we assume user knows what (s)he's doing and we don't + # install modules dependencies + errmsg = '%s is not installed' % basedefs.PUPPET_MODULES_PKG + deps = list(basedefs.PUPPET_DEPENDENCIES) + if errmsg not in modules_deps: + deps.extend([i.strip() for i in modules_deps.split() if i.strip()]) + + details = {} + for hostname in config['HOST_LIST']: + # install Puppet and it's dependencies + server = utils.ScriptRunner(hostname) + packages = ' '.join(deps) + server.append('yum install -y %s' % packages) + server.append('yum update -y %s' % packages) + # yum does not fail if one of the packages is missing + for package in deps: + server.append('rpm -q --whatprovides %s' % package) + server.execute() + + # create the packstack tmp directory server.clear() - server.append("mkdir -p %s" % basedefs.PACKSTACK_VAR_DIR) + server.append('mkdir -p %s' % basedefs.PACKSTACK_VAR_DIR) # Separately create the tmp directory for this packstack run, this will # fail if the directory already exists host_dir = os.path.join(basedefs.PACKSTACK_VAR_DIR, uuid.uuid4().hex) - server.append("mkdir --mode 0700 %s" % host_dir) + server.append('mkdir --mode 0700 %s' % host_dir) for i in ('modules', 'resources'): - server.append("mkdir --mode 0700 %s" % os.path.join(host_dir, i)) + server.append('mkdir --mode 0700 %s' % os.path.join(host_dir, i)) + server.execute() + details.setdefault(hostname, {})['tmpdir'] = host_dir + + # discover other host info; Facter is installed as Puppet dependency, + # so we let it do the work + server.clear() + server.append('facter -p') + rc, stdout = server.execute() + for line in stdout.split('\n'): + try: + key, value = line.split('=>', 1) + except ValueError: + # this line is probably some warning, so let's skip it + continue + else: + details[hostname][key.strip()] = value.strip() + + # create a symbolic link to /etc/hiera.yaml to avoid warning messages + # such as "Warning: Config file /etc/puppet/hiera.yaml not found, + # using Hiera defaults" + server.clear() + server.append('[[ ! -L /etc/puppet/hiera.yaml ]] && ' + 'ln -s /etc/hiera.yaml /etc/puppet/hiera.yaml || ' + 'echo "hiera.yaml symlink already created"') + server.append("sed -i 's;:datadir:.*;:datadir: " + "%s/hieradata;g' /etc/puppet/hiera.yaml" + % details[hostname]['tmpdir']) server.execute() - details[host]['tmpdir'] = host_dir config['HOST_DETAILS'] = details @@ -670,8 +695,9 @@ def create_ntp_manifest(config, messages): marker = uuid.uuid4().hex[:16] for hostname in filtered_hosts(config): - releaseos = config['HOST_DETAILS'][hostname]['os'] - releasever = config['HOST_DETAILS'][hostname]['release'].split('.')[0] + hostnfo = config['HOST_DETAILS'][hostname] + releaseos = hostnfo['operatingsystem'] + releasever = hostnfo['operatingsystemmajrelease'] # Configure chrony for Fedora or RHEL/CentOS 7 if releaseos == 'Fedora' or releasever == '7': diff --git a/packstack/plugins/puppet_950.py b/packstack/plugins/puppet_950.py index d654a968b..6fdaa6ab1 100644 --- a/packstack/plugins/puppet_950.py +++ b/packstack/plugins/puppet_950.py @@ -61,8 +61,6 @@ def initSequences(controller): controller.insertSequence("Clean Up", [], [], puppetpresteps, index=0) puppetsteps = [ - {'title': 'Installing Dependencies', - 'functions': [install_deps]}, {'title': 'Copying Puppet modules and manifests', 'functions': [copy_puppet_modules]}, {'title': 'Applying Puppet manifests', @@ -142,48 +140,6 @@ def run_cleanup(config, messages): localserver.execute() -def install_deps(config, messages): - deps = ["puppet", "hiera", "openssh-clients", "tar", "nc"] - modules_pkg = 'openstack-puppet-modules' - - local = utils.ScriptRunner() - local.append('rpm -q --requires %s | egrep -v "^(rpmlib|\/|perl)"' - % modules_pkg) - - # This can fail if there are no dependencies other than those - # filtered out by the egrep expression. - rc, modules_deps = local.execute(can_fail=False) - - # Modules package might not be installed if we are running from source. - # In this case we assume user knows what (s)he's doing and we don't - # install modules dependencies - if ('%s is not installed' % modules_pkg) not in modules_deps: - modules_deps = [i.strip() for i in modules_deps.split() if i.strip()] - deps.extend(modules_deps) - - for hostname in filtered_hosts(config): - server = utils.ScriptRunner(hostname) - packages = ' '.join(deps) - server.append("yum install -y %s" % packages) - server.append("yum update -y %s" % packages) - # yum does not fail if one of the packages is missing - for package in deps: - server.append("rpm -q --whatprovides %s" % (package)) - - # To avoid warning messages such as - # "Warning: Config file /etc/puppet/hiera.yaml not found, using Hiera - # defaults". We create a symbolic link to /etc/hiera.yaml. - server.append('[[ ! -L /etc/puppet/hiera.yaml ]] && ' - 'ln -s /etc/hiera.yaml /etc/puppet/hiera.yaml || ' - 'echo "hiera.yaml symlink already created"') - - server.append("sed -i 's;:datadir:.*;:datadir: " - "%s/hieradata;g' /etc/puppet/hiera.yaml" - % config['HOST_DETAILS'][hostname]['tmpdir']) - - server.execute() - - def copy_puppet_modules(config, messages): os_modules = ' '.join(('apache', 'ceilometer', 'certmonger', 'cinder', 'concat', 'firewall', 'glance', 'heat', 'horizon', diff --git a/packstack/plugins/serverprep_001.py b/packstack/plugins/serverprep_001.py index abb3db1d1..d9d2240a3 100644 --- a/packstack/plugins/serverprep_001.py +++ b/packstack/plugins/serverprep_001.py @@ -425,7 +425,7 @@ def run_rhsm_reg(host, username, password, optional=False, proxy_server=None, """ Registers given host to Red Hat Repositories via subscription manager. """ - releasever = config['HOST_DETAILS'][host]['release'].split('.')[0] + releasever = config['HOST_DETAILS'][host]['operatingsystemmajrelease'] server = utils.ScriptRunner(host) # configure proxy if it is necessary @@ -477,11 +477,12 @@ def manage_epel(host, config): Installs and/or enables EPEL repo if it is required or disables it if it is not required. """ - if config['HOST_DETAILS'][host]['os'] in ('Fedora', 'Unknown'): + relevant = ('redhat', 'centos', 'scientific') + if config['HOST_DETAILS'][host]['operatingsystem'].lower() not in relevant: return # yum's $releasever can be non numeric on RHEL, so interpolate here - releasever = config['HOST_DETAILS'][host]['release'].split('.')[0] + releasever = config['HOST_DETAILS'][host]['operatingsystemmajrelease'] mirrors = ('https://mirrors.fedoraproject.org/metalink?repo=epel-%s&' 'arch=$basearch' % releasever) server = utils.ScriptRunner(host) diff --git a/tests/installer/test_run_setup.py b/tests/installer/test_run_setup.py index a74ee81d6..4515a19d4 100644 --- a/tests/installer/test_run_setup.py +++ b/tests/installer/test_run_setup.py @@ -65,6 +65,11 @@ class CommandLineTestCase(PackstackTestCaseMixin, TestCase): 'openstack-icehouse', stdout='[openstack-icehouse]\nenabled=1') + FakePopen.register_as_script( + 'facter -p', + stdout='operatingsystem => Fedora\noperatingsystemmajrelease => 21' + ) + # required by packstack.plugins.nova_300.gather_host_keys FakePopen.register('ssh-keyscan 127.0.0.1', stdout='127.0.0.1 ssh-rsa hostkey-data')