From eba1d3d5d207ba1d3b428e21b63fa9ed4456045f Mon Sep 17 00:00:00 2001 From: Martin Magr Date: Fri, 1 Mar 2013 18:30:07 +0100 Subject: [PATCH] Cleaned packstack.installer.common_utils - cleaned from useless functions - refactored to be pep8 compliant - separated to standalone modules - added unit tests Change-Id: If5a300fe04efab9e13549f2b4d05c1b28e2e40c6 --- packstack/installer/basedefs.py | 23 +- packstack/installer/common_utils.py | 431 ------------------ packstack/installer/exceptions.py | 13 +- packstack/installer/output_messages.py | 1 - packstack/installer/processors.py | 4 +- packstack/installer/run_setup.py | 40 +- packstack/installer/setup_params.py | 2 +- packstack/installer/setup_sequences.py | 4 +- packstack/installer/utils/__init__.py | 14 + .../installer/{ => utils}/datastructures.py | 0 packstack/installer/utils/decorators.py | 29 ++ packstack/installer/utils/network.py | 74 +++ packstack/installer/utils/shell.py | 118 +++++ packstack/installer/utils/strings.py | 34 ++ packstack/installer/validators.py | 6 +- packstack/plugins/cinder_250.py | 6 +- packstack/plugins/dashboard_500.py | 6 +- packstack/plugins/glance_200.py | 6 +- packstack/plugins/keystone_100.py | 8 +- packstack/plugins/mysql_001.py | 8 +- packstack/plugins/nagios_910.py | 6 +- packstack/plugins/nova_300.py | 16 +- packstack/plugins/openstack_client_400.py | 6 +- packstack/plugins/postscript_949.py | 4 +- packstack/plugins/prescript_000.py | 4 +- packstack/plugins/puppet_950.py | 8 +- packstack/plugins/qpid_002.py | 6 +- packstack/plugins/serverprep_901.py | 6 +- packstack/plugins/sshkeys_000.py | 6 +- packstack/plugins/swift_600.py | 8 +- tests/installer/test_utils.py | 93 ++++ 31 files changed, 453 insertions(+), 537 deletions(-) delete mode 100644 packstack/installer/common_utils.py create mode 100644 packstack/installer/utils/__init__.py rename packstack/installer/{ => utils}/datastructures.py (100%) create mode 100644 packstack/installer/utils/decorators.py create mode 100644 packstack/installer/utils/network.py create mode 100644 packstack/installer/utils/shell.py create mode 100644 packstack/installer/utils/strings.py create mode 100644 tests/installer/test_utils.py diff --git a/packstack/installer/basedefs.py b/packstack/installer/basedefs.py index eabf0faf8..f51451344 100644 --- a/packstack/installer/basedefs.py +++ b/packstack/installer/basedefs.py @@ -1,11 +1,15 @@ +# -*- coding: utf-8 -*- + """ -provides all the predefined variables for engine-setup +This module provides all the predefined variables. """ + import os import sys import datetime import tempfile + APP_NAME = "Installer" FILE_YUM_VERSION_LOCK = "/etc/yum/pluginconf.d/versionlock.list" @@ -17,7 +21,7 @@ except: pass _tmpdirprefix = datetime.datetime.now().strftime('%Y%m%d-%H%M%S-') -VAR_DIR = tempfile.mkdtemp(prefix = _tmpdirprefix, dir = PACKSTACK_VAR_DIR) +VAR_DIR = tempfile.mkdtemp(prefix=_tmpdirprefix, dir=PACKSTACK_VAR_DIR) DIR_LOG = VAR_DIR PUPPET_MANIFEST_RELATIVE = "manifests" PUPPET_MANIFEST_DIR = os.path.join(VAR_DIR, PUPPET_MANIFEST_RELATIVE) @@ -37,17 +41,10 @@ EXEC_CHKCONFIG = "chkconfig" EXEC_SERVICE = "service" EXEC_IP = "ip" -#text colors -RED = "\033[0;31m" -GREEN = "\033[92m" -BLUE = "\033[94m" -YELLOW = "\033[93m" +# text colors NO_COLOR = "\033[0m" +COLORS = {'red': "\033[0;31m", 'green': "\033[92m", 'blue': "\033[94m", + 'yellow': "\033[93m"} -COLORS = (RED, GREEN, BLUE, YELLOW, NO_COLOR) - -#space len size for color print +# space len size for color print SPACE_LEN = 70 - -RPM_LOCK_LIST = """ -""" diff --git a/packstack/installer/common_utils.py b/packstack/installer/common_utils.py deleted file mode 100644 index 577c5adb1..000000000 --- a/packstack/installer/common_utils.py +++ /dev/null @@ -1,431 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -contains all common and re-usable code for rhevm-setup and sub packages -""" -import grp -import pwd -import logging -import subprocess -import re -import traceback -import os -import datetime -import types -import time -import socket -import tempfile - -import basedefs -import output_messages -from .exceptions import NetworkError, ScriptRuntimeError - - - -def getColoredText (text, color): - ''' gets text string and color - and returns a colored text. - the color values are RED/BLUE/GREEN/YELLOW - everytime we color a text, we need to disable - the color at the end of it, for that - we use the NO_COLOR chars. - ''' - return color + text + basedefs.NO_COLOR - -def execCmd(cmdList, cwd=None, failOnError=False, msg=output_messages.ERR_RC_CODE, maskList=[], useShell=False, usePipeFiles=False): - """ - Run external shell command with 'shell=false' - receives a list of arguments for command line execution - """ - # All items in the list needs to be strings, otherwise the subprocess will fail - cmd = [str(item) for item in cmdList] - - # We need to join cmd list into one string so we can look for passwords in it and mask them - logCmd = _maskString((' '.join(cmd)), maskList) - logging.debug("Executing command --> '%s'"%(logCmd)) - - stdErrFD = subprocess.PIPE - stdOutFD = subprocess.PIPE - stdInFD = subprocess.PIPE - - if usePipeFiles: - (stdErrFD, stdErrFile) = tempfile.mkstemp(dir="/tmp") - (stdOutFD, stdOutFile) = tempfile.mkstemp(dir="/tmp") - (stdInFD, stdInFile) = tempfile.mkstemp(dir="/tmp") - - # We use close_fds to close any file descriptors we have so it won't be copied to forked childs - proc = subprocess.Popen(cmd, stdout=stdOutFD, - stderr=stdErrFD, stdin=stdInFD, cwd=cwd, shell=useShell, close_fds=True) - - out, err = proc.communicate() - if usePipeFiles: - with open(stdErrFile, 'r') as f: - err = f.read() - os.remove(stdErrFile) - - with open(stdOutFile, 'r') as f: - out = f.read() - os.remove(stdOutFile) - os.remove(stdInFile) - - logging.debug("output = %s"%(out)) - logging.debug("stderr = %s"%(err)) - logging.debug("retcode = %s"%(proc.returncode)) - output = out + err - if failOnError and proc.returncode != 0: - raise Exception(msg) - return ("".join(output.splitlines(True)), proc.returncode) - -def byLength(word1, word2): - """ - Compars two strings by their length - Returns: - Negative if word2 > word1 - Positive if word1 > word2 - Zero if word1 == word 2 - """ - return len(word1) - len(word2) - -def nslookup(address): - cmd = [ - basedefs.EXEC_NSLOOKUP, address, - ] - #since nslookup will return 0 no matter what, the RC is irrelevant - output, rc = execCmd(cmdList=cmd) - return output - -def getConfiguredIps(): - try: - iplist=set() - cmd = [ - basedefs.EXEC_IP, "addr", - ] - output, rc = execCmd(cmdList=cmd, failOnError=True, msg=output_messages.ERR_EXP_GET_CFG_IPS_CODES) - ipaddrPattern=re.compile('\s+inet (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}).+') - list=output.splitlines() - for line in list: - foundIp = ipaddrPattern.search(line) - if foundIp: - if foundIp.group(1) != "127.0.0.1": - ipAddr = foundIp.group(1) - logging.debug("Found IP Address: %s"%(ipAddr)) - iplist.add(ipAddr) - return iplist - except: - logging.error(traceback.format_exc()) - raise Exception(output_messages.ERR_EXP_GET_CFG_IPS) - -def getCurrentDateTime(isUtc=None): - now = None - if (isUtc is not None): - now = datetime.datetime.utcnow() - else: - now = datetime.datetime.now() - return now.strftime("%Y_%m_%d_%H_%M_%S") - -def verifyStringFormat(str, matchRegex): - ''' - Verify that the string given matches the matchRegex. - for example: - string: 111-222 - matchRegex: \d{3}-\d{3} - this will return true since the string matches the regex - ''' - pattern = re.compile(matchRegex) - result = re.match(pattern, str) - if result == None: - return False - else: - return True - -def compareStrIgnoreCase(str1, str2): - ''' compare 2 strings and ignore case - if one of the input is not str (bool for e.g) - return normal comapre - ''' - if type(str1) == types.StringType and type(str2) == types.StringType: - return str1.lower() == str2.lower() - else: - return str1 == str2 - -def parseStrRegex(string, regex, errMsg): - """ - Gets a text string and a regex pattern - and returns the extracted sub-string - captured. - """ - rePattern = re.compile(regex) - found = rePattern.search(string) - if found: - match = found.group(1) - logging.debug("found new parsed string: %s"%(match)) - return match - else: - raise Exception(errMsg) - -def _maskString(string, maskList=[]): - """ - private func to mask passwords - in utils - """ - maskedStr = string - for maskItem in maskList: - if not maskItem: continue - maskedStr = maskedStr.replace(maskItem, "*"*8) - # if looking at stderr of a script, single quotes have been converted - # to '\'' - maskedStr = maskedStr.replace(maskItem.replace("'","'\\''"), "*"*8) - - return maskedStr - -def retry(func, expectedException=Exception, tries=None, timeout=None, sleep=1): - """ - Retry a function. Wraps the retry logic so you don't have to - implement it each time you need it. - - :param func: The callable to run. - :param expectedException: The exception you expect to receive when the function fails. - :param tries: The number of time to try. None\0,-1 means infinite. - :param timeout: The time you want to spend waiting. This **WILL NOT** stop the method. - It will just not run it if it ended after the timeout. - :param sleep: Time to sleep between calls in seconds. - """ - if tries in [0, None]: - tries = -1 - - if timeout in [0, None]: - timeout = -1 - - startTime = time.time() - - while True: - tries -= 1 - try: - return func() - except expectedException: - if tries == 0: - raise - - if (timeout > 0) and ((time.time() - startTime) > timeout): - raise - - time.sleep(sleep) - -def localHost(hostname): - # Create an ip set of possible IPs on the machine. Set has only unique values, so - # there's no problem with union. - # TODO: cache the list somehow? There's no poing quering the IP configuraion all the time. - ipset = getConfiguredIps().union(set([ "localhost", "127.0.0.1"])) - if hostname in ipset: - return True - return False - -# TODO: Support SystemD services -class Service(): - def __init__(self, name): - self.wasStopped = False - self.wasStarted = False - self.name = name - - def isServiceAvailable(self): - if os.path.exists("/etc/init.d/%s" % self.name): - return True - return False - - def start(self, raiseFailure = False): - logging.debug("starting %s", self.name) - (output, rc) = self._serviceFacility("start") - if rc == 0: - self.wasStarted = True - elif raiseFailure: - raise Exception(output_messages.ERR_FAILED_START_SERVICE % self.name) - - return (output, rc) - - def stop(self, raiseFailure = False): - logging.debug("stopping %s", self.name) - (output, rc) = self._serviceFacility("stop") - if rc == 0: - self.wasStopped = True - elif raiseFailure: - raise Exception(output_messages.ERR_FAILED_STOP_SERVICE % self.name) - - return (output, rc) - - def autoStart(self, start=True): - mode = "on" if start else "off" - cmd = [ - basedefs.EXEC_CHKCONFIG, self.name, mode, - ] - execCmd(cmdList=cmd, failOnError=True) - - def conditionalStart(self, raiseFailure = False): - """ - Will only start if wasStopped is set to True - """ - if self.wasStopped: - logging.debug("Service %s was stopped. starting it again"%self.name) - return self.start(raiseFailure) - else: - logging.debug("Service was not stopped. there for we're not starting it") - return (False, False) - - def status(self): - logging.debug("getting status for %s", self.name) - (output, rc) = self._serviceFacility("status") - return (output, rc) - - def _serviceFacility(self, action): - """ - Execute the command "service NAME action" - returns: output, rc - """ - logging.debug("executing action %s on service %s", self.name, action) - cmd = [ - basedefs.EXEC_SERVICE, self.name, action - ] - return execCmd(cmdList=cmd, usePipeFiles=True) - -def chown(target,uid, gid): - logging.debug("chown %s to %s:%s" % (target, uid, gid)) - os.chown(target, uid, gid) - -def installed(rpm): - cmd = [ - basedefs.EXEC_RPM, - "-q", - rpm, - ] - output, rc = execCmd(cmd) - return rc == 0 - -def returnYes(controller): - return "yes" - -def getLocalhostIP(): - """ - Returns IP address of localhost. - """ - # TO-DO: Will probably need to find better way to find out localhost - # address. - - # find nameservers - ns_regex = re.compile('nameserver\s*(?P[\d\.\:])') - resolv, rc = execCmd(['cat /etc/resolv.conf | grep nameserver',], - failOnError=False, useShell=True) - nsrvs = [] - for line in resolv.split('\n'): - match = ns_regex.match(line.strip()) - if match: - nsrvs.append(match.group('ns_ip')) - - # try to connect to nameservers and return own IP address - nsrvs.append('8.8.8.8') # default to google dns - for i in nsrvs: - try: - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - s.connect((i, 0)) - loc_ip = s.getsockname()[0] - except socket.error: - continue - else: - return loc_ip - -def host2ip(hostname, allow_localhost=False): - """ - Converts given hostname to IP address. Raises NetworkError - if conversion failed. - """ - try: - ip_list = socket.gethostbyaddr(hostname)[2] - if allow_localhost: - return ip_list[0] - else: - local_ips = ('127.0.0.1', '::1') - for ip in ip_list: - if ip not in local_ips: - break - else: - raise NameError() - return ip - except NameError: - # given hostname is localhost, return appropriate IP address - ip = getLocalhostIP() - if not ip: - raise NetworkError('Failed to get local IP address.') - return ip - except socket.error: - raise NetworkError('Unknown hostname %s.' % hostname) - except Exception, ex: - raise NetworkError('Unknown error appeared: %s' % repr(ex)) - -def forceIP(host, allow_localhost=False): - host = host.strip() - ipv4_regex = re.compile('\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}') - ipv6_regex = re.compile('[abcdef\d\:]+') - if not ipv4_regex.match(host) or not ipv6_regex.match(host): - host = host2ip(host, allow_localhost=allow_localhost) - return host - -def device_from_ip(ip): - server = ScriptRunner() - server.append("DEVICE=$(ip address show to %s | head -n 1 | sed -e 's/.*: \(.*\):.*/\\1/g')" % ip) - # Test device, raises an exception is it doesn't exist - server.append("ip link show \"$DEVICE\" > /dev/null") - server.append("echo $DEVICE") - rv, stdout = server.execute() - return stdout.strip() - -class ScriptRunner(object): - def __init__(self, ip=None): - self.script = [] - self.ip = ip - - def append(self, s): - self.script.append(s) - - def clear(self): - self.script = [] - - def execute(self, logerrors=True, maskList=None): - maskList = maskList or [] - script = "\n".join(self.script) - logging.debug("# ============ ssh : %r =========="%self.ip) - - _PIPE = subprocess.PIPE # pylint: disable=E1101 - if self.ip: - cmd = ["ssh", "-o", "StrictHostKeyChecking=no", - "-o", "UserKnownHostsFile=/dev/null", - "root@%s" % self.ip, "bash -x"] - else: - cmd = ["bash", "-x"] - obj = subprocess.Popen(cmd, stdin=_PIPE, stdout=_PIPE, stderr=_PIPE, - close_fds=True, shell=False) - - logging.debug(_maskString(script, maskList)) - script = "function t(){ exit $? ; } \n trap t ERR \n" + script - stdoutdata, stderrdata = obj.communicate(script) - logging.debug("============ STDOUT ==========") - logging.debug(_maskString(stdoutdata, maskList)) - returncode = obj.returncode - if returncode: - if logerrors: - logging.error("============= STDERR ==========") - logging.error(_maskString(stderrdata, maskList)) - - pattern = (r'^ssh\:') - if re.search(pattern, stderrdata): - raise NetworkError(stderrdata) - else: - raise ScriptRuntimeError('Error running remote script: ' - '%s' % stdoutdata) - return returncode, stdoutdata - - def template(self, src, dst, varsdict): - with open(src) as fp: - self.append("cat > %s <<- EOF\n%s\nEOF\n"%(dst, fp.read()%varsdict)) - - def ifnotexists(self, fn, s): - self.append("[ -e %s ] || %s"%(fn, s)) - - def ifexists(self, fn, s): - self.append("[ -e %s ] && %s"%(fn, s)) diff --git a/packstack/installer/exceptions.py b/packstack/installer/exceptions.py index 0cf23c6fc..aa6c80292 100644 --- a/packstack/installer/exceptions.py +++ b/packstack/installer/exceptions.py @@ -18,7 +18,11 @@ __all__ = ( class PackStackError(Exception): """Default Exception class for packstack installer.""" - pass + def __init__(self, *args, **kwargs): + super(PackStackError, self).__init__(*args) + self.stdout = kwargs.get('stdout', None) + self.stderr = kwargs.get('stderr', None) + class MissingRequirements(PackStackError): @@ -55,5 +59,10 @@ class NetworkError(PackStackError): class ScriptRuntimeError(PackStackError): - """Raised when ScriptRunner.execute does not end successfully.""" + """ + Raised when utils.ScriptRunner.execute does not end successfully. + """ pass + +class ExecuteRuntimeError(PackStackError): + """Raised when utils.execute does not end successfully.""" diff --git a/packstack/installer/output_messages.py b/packstack/installer/output_messages.py index fc5bcfcc4..c60616e0b 100644 --- a/packstack/installer/output_messages.py +++ b/packstack/installer/output_messages.py @@ -77,7 +77,6 @@ ERR_IPS_NOT_CONFIGED_ON_INT="The IP (%s) which was resolved from the FQDN %s is ERR_IPS_HAS_NO_PTR="None of the IP addresses on this host(%s) holds a PTR record for the FQDN: %s" ERR_IP_HAS_NO_PTR="The IP %s does not hold a PTR record for the FQDN: %s" ERR_EXP_FAILED_INIT_LOGGER="Unexpected error: Failed to initiate logger, please check file system permission" -ERR_RC_CODE="Return Code is not zero" ERR_FAILURE="General failure" ERR_NO_ANSWER_FILE="Error: Could not find file %s" ERR_ONLY_1_FLAG="Error: The %s flag is mutually exclusive to all other command line options" diff --git a/packstack/installer/processors.py b/packstack/installer/processors.py index 919314078..1355452a4 100644 --- a/packstack/installer/processors.py +++ b/packstack/installer/processors.py @@ -2,7 +2,7 @@ import os -from .common_utils import ScriptRunner, forceIP +from .utils import ScriptRunner, force_ip from .exceptions import ParamProcessingError, NetworkError @@ -17,7 +17,7 @@ def process_host(param, process_args=None): localhost = process_args and \ process_args.get('allow_localhost', False) try: - return forceIP(param, allow_localhost=localhost) + return force_ip(param, allow_localhost=localhost) except NetworkError, ex: raise ParamProcessingError(str(ex)) diff --git a/packstack/installer/run_setup.py b/packstack/installer/run_setup.py index d5d211683..8c241d154 100644 --- a/packstack/installer/run_setup.py +++ b/packstack/installer/run_setup.py @@ -15,7 +15,7 @@ from optparse import OptionParser, OptionGroup import basedefs import validators -import common_utils as utils +from . import utils import processors import output_messages from .exceptions import FlagValidationError, ParamValidationError @@ -35,7 +35,6 @@ def initLogging (debug): global logFile try: - #in order to use UTC date for the log file, send True to getCurrentDateTime(True) logFilename = "openstack-setup.log" logFile = os.path.join(basedefs.DIR_LOG, logFilename) @@ -220,18 +219,19 @@ def mask(input): if type(input) == types.DictType: for key in input: if type(input[key]) == types.StringType: - output[key] = maskString(input[key]) + output[key] = utils.mask_string(input[key], + masked_value_set) if type(input) == types.ListType: for item in input: org = item orgIndex = input.index(org) if type(item) == types.StringType: - item = maskString(item) + item = utils.mask_string(item, masked_value_set) if item != org: output.remove(org) output.insert(orgIndex, item) if type(input) == types.StringType: - output = maskString(input) + output = utils.mask_string(input, masked_value_set) return output @@ -253,13 +253,6 @@ def removeMaskString(maskedString): if found: masked_value_set.remove(maskedString) -def maskString(str): - # Iterate sorted list, so we won't mask only part of a password - for password in sorted(masked_value_set, utils.byLength, None, True): - if password: - str = str.replace(password, '*'*8) - return str - def validate_param_value(param, value): cname = param.CONF_NAME logging.debug("Validating parameter %s." % cname) @@ -364,7 +357,7 @@ def _handleAnswerFileParams(answerFile): # Handle pre condition match with case insensitive values logging.info("Comparing pre- conditions, value: '%s', and match: '%s'" % (preConditionValue, group.PRE_CONDITION_MATCH)) - if utils.compareStrIgnoreCase(preConditionValue, group.PRE_CONDITION_MATCH): + if preConditionValue == group.PRE_CONDITION_MATCH: for param in group.parameters.itervalues(): _loadParamFromFile(fconf, "general", param.CONF_NAME) @@ -374,7 +367,7 @@ def _handleAnswerFileParams(answerFile): postConditionValue = _handleGroupCondition(fconf, group.POST_CONDITION, postConditionValue) # Handle post condition match for group - if not utils.compareStrIgnoreCase(postConditionValue, group.POST_CONDITION_MATCH): + if postConditionValue != group.POST_CONDITION_MATCH: logging.error("The group condition (%s) returned: %s, which differs from the excpeted output: %s"%\ (group.GROUP_NAME, postConditionValue, group.POST_CONDITION_MATCH)) raise ValueError(output_messages.ERR_EXP_GROUP_VALIDATION_ANS_FILE%\ @@ -422,7 +415,7 @@ def _handleInteractiveParams(): # If we have a match, i.e. condition returned True, go over all params in the group logging.info("Comparing pre-conditions; condition: '%s', and match: '%s'" % (preConditionValue, group.PRE_CONDITION_MATCH)) - if utils.compareStrIgnoreCase(preConditionValue, group.PRE_CONDITION_MATCH): + if preConditionValue == group.PRE_CONDITION_MATCH: while inputLoop: for param in group.parameters.itervalues(): if not param.CONDITION: @@ -546,19 +539,6 @@ def _addFinalInfoMsg(): """ controller.MESSAGES.append(output_messages.INFO_LOG_FILE_PATH%(logFile)) -def _lockRpmVersion(): - """ - Enters rpm versions into yum version-lock - """ - logging.debug("Locking rpms in yum-version-lock") - cmd = [ - basedefs.EXEC_RPM, "-q", - ] + basedefs.RPM_LOCK_LIST.split() - output, rc = utils.execCmd(cmdList=cmd, failOnError=True, msg=output_messages.ERR_YUM_LOCK) - - with open(basedefs.FILE_YUM_VERSION_LOCK, "a") as f: - for rpm in output.splitlines(): - f.write(rpm + "\n") def _summaryParamsToLog(): if len(controller.CONF) > 0: @@ -624,7 +604,7 @@ def remove_remote_var_dirs(): msg = output_messages.ERR_REMOVE_REMOTE_VAR % (host_dir, host) logging.error(msg) logging.exception(e) - controller.MESSAGES.append(utils.getColoredText(msg, basedefs.RED)) + controller.MESSAGES.append(utils.color_text(msg, 'red')) def generateAnswerFile(outputFile, overrides={}): @@ -905,7 +885,7 @@ def main(): except Exception as e: logging.error(traceback.format_exc()) print - print utils.getColoredText("ERROR : "+str(e), basedefs.RED) + print utils.color_text("ERROR : " + str(e), 'red') print output_messages.ERR_CHECK_LOG_FILE_FOR_MORE_INFO%(logFile) sys.exit(1) diff --git a/packstack/installer/setup_params.py b/packstack/installer/setup_params.py index 90cf55cf3..b2a8a8079 100644 --- a/packstack/installer/setup_params.py +++ b/packstack/installer/setup_params.py @@ -4,7 +4,7 @@ Container set for groups and parameters """ -from .datastructures import SortedDict +from .utils.datastructures import SortedDict class Parameter(object): diff --git a/packstack/installer/setup_sequences.py b/packstack/installer/setup_sequences.py index 9aac5f1cf..159bf404c 100644 --- a/packstack/installer/setup_sequences.py +++ b/packstack/installer/setup_sequences.py @@ -8,7 +8,7 @@ import string import traceback import basedefs import output_messages -import common_utils as utils +from . import utils class Step(object): def __init__(self, title=None, functions=[]): @@ -61,7 +61,7 @@ class Step(object): except: logging.debug(traceback.format_exc()) raise - print ("[ " + utils.getColoredText(output_messages.INFO_DONE, basedefs.GREEN) + " ]").rjust(spaceLen) + print ("[ " + utils.color_text(output_messages.INFO_DONE, 'green') + " ]").rjust(spaceLen) class Sequence(object): """ diff --git a/packstack/installer/utils/__init__.py b/packstack/installer/utils/__init__.py new file mode 100644 index 000000000..a380c9780 --- /dev/null +++ b/packstack/installer/utils/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- + +from .datastructures import SortedDict +from .decorators import retry +from .network import get_localhost_ip, host2ip, force_ip +from .shell import ScriptRunner, execute +from .strings import color_text, mask_string + + +__all__ = ('SortedDict', + 'retry', + 'ScriptRunner', 'execute', + 'get_localhost_ip', 'host2ip', 'force_ip', + 'color_text', 'mask_string') diff --git a/packstack/installer/datastructures.py b/packstack/installer/utils/datastructures.py similarity index 100% rename from packstack/installer/datastructures.py rename to packstack/installer/utils/datastructures.py diff --git a/packstack/installer/utils/decorators.py b/packstack/installer/utils/decorators.py new file mode 100644 index 000000000..1f9a1fdf7 --- /dev/null +++ b/packstack/installer/utils/decorators.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +import time + + +def retry(count=1, delay=0, retry_on=Exception): + """ + Decorator which tries to run specified fuction if the previous + run ended by given exception. Retry count and delays can be also + specified. + """ + if count < 0 or delay < 0: + raise ValueError('Count and delay has to be positive number.') + + def decorator(func): + def wrapper(*args, **kwargs): + tried = 0 + while tried <= count: + try: + return func(*args, **kwargs) + except retry_on: + if tried >= count: + raise + if delay: + time.sleep(delay) + tried += 1 + wrapper.func_name = func.func_name + return wrapper + return decorator diff --git a/packstack/installer/utils/network.py b/packstack/installer/utils/network.py new file mode 100644 index 000000000..22c2aa5a3 --- /dev/null +++ b/packstack/installer/utils/network.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- + +import re +import socket + +from .shell import execute + + +def get_localhost_ip(): + """ + Returns IP address of localhost. + """ + # TO-DO: Will probably need to find better way to find out localhost + # address. + + # find nameservers + ns_regex = re.compile('nameserver\s*(?P[\d\.\:])') + rc, resolv = execute('cat /etc/resolv.conf | grep nameserver', + can_fail=False, use_shell=True) + nsrvs = [] + for line in resolv.split('\n'): + match = ns_regex.match(line.strip()) + if match: + nsrvs.append(match.group('ns_ip')) + + # try to connect to nameservers and return own IP address + nsrvs.append('8.8.8.8') # default to google dns + for i in nsrvs: + try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect((i, 0)) + loc_ip = s.getsockname()[0] + except socket.error: + continue + else: + return loc_ip + + +def host2ip(hostname, allow_localhost=False): + """ + Converts given hostname to IP address. Raises NetworkError + if conversion failed. + """ + try: + ip_list = socket.gethostbyaddr(hostname)[2] + if allow_localhost: + return ip_list[0] + else: + local_ips = ('127.0.0.1', '::1') + for ip in ip_list: + if ip not in local_ips: + break + else: + raise NameError() + return ip + except NameError: + # given hostname is localhost, return appropriate IP address + ip = get_localhost_ip() + if not ip: + raise NetworkError('Failed to get local IP address.') + return ip + except socket.error: + raise NetworkError('Unknown hostname %s.' % hostname) + except Exception, ex: + raise NetworkError('Unknown error appeared: %s' % repr(ex)) + + +def force_ip(host, allow_localhost=False): + host = host.strip() + ipv4_regex = re.compile('\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}') + ipv6_regex = re.compile('[abcdef\d\:]+') + if not ipv4_regex.match(host) or not ipv6_regex.match(host): + host = host2ip(host, allow_localhost=allow_localhost) + return host diff --git a/packstack/installer/utils/shell.py b/packstack/installer/utils/shell.py new file mode 100644 index 000000000..787a7f496 --- /dev/null +++ b/packstack/installer/utils/shell.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- + +import re +import types +import logging +import subprocess + +from ..exceptions import (ExecuteRuntimeError, ScriptRuntimeError, + NetworkError) +from .strings import mask_string + + +def execute(cmd, workdir=None, can_fail=False, mask_list=None, + use_shell=False): + """ + Runs shell command cmd. If can_fail is set to False + ExecuteRuntimeError is raised if command returned non-zero return + code. Otherwise + """ + mask_list = mask_list or [] + repl_list = [("'","'\\''")] + + if type(cmd) is not types.StringType: + import pipes + masked = ' '.join((pipes.quote(i) for i in cmd)) + else: + masked = cmd + masked = mask_string(masked, mask_list, repl_list) + logging.debug("Executing command: %s" % masked) + + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, cwd=workdir, + shell=use_shell, close_fds=True) + out, err = proc.communicate() + logging.debug("rc: %s" % proc.returncode) + logging.debug("stdout:\n%s" % mask_string(out, mask_list, repl_list)) + logging.debug("stderr:\n%s" % mask_string(err, mask_list, repl_list)) + + if not can_fail and proc.returncode != 0: + msg = 'Failed to execute command: %s' % masked + raise ExecuteRuntimeError(msg, stdout=out, stderr=err) + return proc.returncode, out + + +class ScriptRunner(object): + _pkg_search = 'rpm -q' + + def __init__(self, ip=None): + self.script = [] + self.ip = ip + + def append(self, s): + self.script.append(s) + + def clear(self): + self.script = [] + + def execute(self, logerrors=True, maskList=None): + maskList = maskList or [] + script = "\n".join(self.script) + logging.debug("# ============ ssh : %r =========="%self.ip) + + _PIPE = subprocess.PIPE # pylint: disable=E1101 + if self.ip: + cmd = ["ssh", "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "root@%s" % self.ip, "bash -x"] + else: + cmd = ["bash", "-x"] + obj = subprocess.Popen(cmd, stdin=_PIPE, stdout=_PIPE, + stderr=_PIPE, close_fds=True, + shell=False) + + replace_list = [("'","'\\''")] + logging.debug(mask_string(script, maskList, replace_list)) + script = "function t(){ exit $? ; } \n trap t ERR \n" + script + stdoutdata, stderrdata = obj.communicate(script) + logging.debug("============ STDOUT ==========") + logging.debug(mask_string(stdoutdata, maskList, replace_list)) + returncode = obj.returncode + if returncode: + if logerrors: + logging.error("============= STDERR ==========") + logging.error(mask_string(stderrdata, maskList, + replace_list)) + + pattern = (r'^ssh\:') + if re.search(pattern, stderrdata): + raise NetworkError(stderrdata, stdout=stdoutdata, + stderr=stderrdata) + else: + msg = 'Error running remote script: %s' % stdoutdata + raise ScriptRuntimeError(msg, stdout=stdoutdata, + stderr=stderrdata) + return returncode, stdoutdata + + def template(self, src, dst, varsdict): + with open(src) as fp: + content = fp.read() % varsdict + self.append("cat > %s <<- EOF\n%s\nEOF\n" % (dst, content)) + + def if_not_exists(self, path, command): + self.append("[ -e %s ] || %s" % (path, command)) + + def if_exists(self, path, command): + self.append("[ -e %s ] && %s" % (path, command)) + + def if_installed(self, pkg, command): + self.append("%s %s && %s" % (self._pkg_search, pkg, command)) + + def if_not_installed(self, pkg, command): + self.append("%s %s || %s" % (self._pkg_search, pkg, command)) + + def chown(self, target, uid, gid): + self.append("chown %s:%s %s" % (uid, gid, target)) + + def chmod(self, target, mode): + self.append("chown %s %s" % (mode, target)) diff --git a/packstack/installer/utils/strings.py b/packstack/installer/utils/strings.py new file mode 100644 index 000000000..af47325c2 --- /dev/null +++ b/packstack/installer/utils/strings.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +from .. import basedefs + + +STR_MASK = '*' * 8 + + +def color_text(text, color): + """ + Returns given text string with appropriate color tag. Allowed values + for color parameter are 'red', 'blue', 'green' and 'yellow'. + """ + return '%s%s%s' % (basedefs.COLORS[color], text, basedefs.NO_COLOR) + + +def mask_string(unmasked, mask_list=None, replace_list=None): + """ + Replaces words from mask_list with MASK in unmasked string. + If words are needed to be transformed before masking, transformation + could be describe in replace list. For example [("'","'\\''")] + replaces all ' characters with '\\''. + """ + mask_list = mask_list or [] + replace_list = replace_list or [] + + masked = unmasked + for word in sorted(mask_list, lambda x, y: len(y) - len(x)): + if not word: + continue + for before, after in replace_list: + word = word.replace(before, after) + masked = masked.replace(word, STR_MASK) + return masked diff --git a/packstack/installer/validators.py b/packstack/installer/validators.py index 693885630..149b414a0 100644 --- a/packstack/installer/validators.py +++ b/packstack/installer/validators.py @@ -12,7 +12,7 @@ import tempfile import traceback import basedefs -import common_utils as utils +from . import utils from .setup_controller import Controller from .exceptions import ParamValidationError @@ -175,8 +175,8 @@ def validate_ping(param, options=None): # TO-DO: to be more flexible, remove this and exit in case param is empty validate_not_empty(param) - cmd = ["/bin/ping", "-c", "1", str(param)] - out, rc = utils.execCmd(cmdList=cmd) + rc, out = utils.execute(['/bin/ping', '-c', '1', str(param)], + can_fail=True) if rc != 0: logging.debug('validate_ping(%s, options=%s) failed.' % (param, options)) diff --git a/packstack/plugins/cinder_250.py b/packstack/plugins/cinder_250.py index 5996d0aad..c0b0c0290 100644 --- a/packstack/plugins/cinder_250.py +++ b/packstack/plugins/cinder_250.py @@ -10,7 +10,7 @@ from packstack.installer import exceptions from packstack.installer import validators from packstack.installer import basedefs -import packstack.installer.common_utils as utils +from packstack.installer import utils from packstack.modules.ospluginutils import getManifestTemplate, appendManifestFile from packstack.installer.exceptions import ScriptRuntimeError @@ -22,7 +22,7 @@ controller = None # Plugin name PLUGIN_NAME = "OS-Cinder" -PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE) +PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue') logging.debug("plugin %s loaded", __name__) @@ -37,7 +37,7 @@ def initConfig(controllerObject): "PROMPT" : "Enter the IP address of the Cinder server", "OPTION_LIST" : [], "VALIDATORS" : [validators.validate_ssh], - "DEFAULT_VALUE" : utils.getLocalhostIP(), + "DEFAULT_VALUE" : utils.get_localhost_ip(), "MASK_INPUT" : False, "LOOSE_VALIDATION": True, "CONF_NAME" : "CONFIG_CINDER_HOST", diff --git a/packstack/plugins/dashboard_500.py b/packstack/plugins/dashboard_500.py index 1515f4e70..03ec7b22b 100644 --- a/packstack/plugins/dashboard_500.py +++ b/packstack/plugins/dashboard_500.py @@ -9,7 +9,7 @@ import uuid from packstack.installer import validators from packstack.installer import basedefs, output_messages from packstack.installer import exceptions -import packstack.installer.common_utils as utils +from packstack.installer import utils from packstack.modules.ospluginutils import getManifestTemplate, appendManifestFile @@ -18,7 +18,7 @@ controller = None # Plugin name PLUGIN_NAME = "OS-HORIZON" -PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE) +PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue') logging.debug("plugin %s loaded", __name__) @@ -32,7 +32,7 @@ def initConfig(controllerObject): "PROMPT" : "Enter the IP address of the Horizon server", "OPTION_LIST" : [], "VALIDATORS" : [validators.validate_ssh], - "DEFAULT_VALUE" : utils.getLocalhostIP(), + "DEFAULT_VALUE" : utils.get_localhost_ip(), "MASK_INPUT" : False, "LOOSE_VALIDATION": True, "CONF_NAME" : "CONFIG_HORIZON_HOST", diff --git a/packstack/plugins/glance_200.py b/packstack/plugins/glance_200.py index 41458ce22..804b49e77 100644 --- a/packstack/plugins/glance_200.py +++ b/packstack/plugins/glance_200.py @@ -7,7 +7,7 @@ import logging from packstack.installer import validators from packstack.installer import basedefs -import packstack.installer.common_utils as utils +from packstack.installer import utils from packstack.modules.ospluginutils import getManifestTemplate, appendManifestFile @@ -16,7 +16,7 @@ controller = None # Plugin name PLUGIN_NAME = "OS-Glance" -PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE) +PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue') logging.debug("plugin %s loaded", __name__) @@ -30,7 +30,7 @@ def initConfig(controllerObject): "PROMPT" : "Enter the IP address of the Glance server", "OPTION_LIST" : [], "VALIDATORS" : [validators.validate_ssh], - "DEFAULT_VALUE" : utils.getLocalhostIP(), + "DEFAULT_VALUE" : utils.get_localhost_ip(), "MASK_INPUT" : False, "LOOSE_VALIDATION": True, "CONF_NAME" : "CONFIG_GLANCE_HOST", diff --git a/packstack/plugins/keystone_100.py b/packstack/plugins/keystone_100.py index 5c6448044..8e6273b3c 100644 --- a/packstack/plugins/keystone_100.py +++ b/packstack/plugins/keystone_100.py @@ -8,7 +8,7 @@ import uuid from packstack.installer import validators from packstack.installer import basedefs -import packstack.installer.common_utils as utils +from packstack.installer import utils from packstack.modules.ospluginutils import getManifestTemplate, appendManifestFile @@ -17,7 +17,7 @@ controller = None # Plugin name PLUGIN_NAME = "OS-Keystone" -PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE) +PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue') logging.debug("plugin %s loaded", __name__) @@ -31,7 +31,7 @@ def initConfig(controllerObject): "PROMPT" : "Enter the IP address of the Keystone server", "OPTION_LIST" : [], "VALIDATORS" : [validators.validate_ssh], - "DEFAULT_VALUE" : utils.getLocalhostIP(), + "DEFAULT_VALUE" : utils.get_localhost_ip(), "MASK_INPUT" : False, "LOOSE_VALIDATION": True, "CONF_NAME" : "CONFIG_KEYSTONE_HOST", @@ -78,7 +78,7 @@ def initConfig(controllerObject): groupDict = { "GROUP_NAME" : "KEYSTONE", "DESCRIPTION" : "Keystone Config parameters", - "PRE_CONDITION" : utils.returnYes, + "PRE_CONDITION" : lambda x: 'yes', "PRE_CONDITION_MATCH" : "yes", "POST_CONDITION" : False, "POST_CONDITION_MATCH" : True} diff --git a/packstack/plugins/mysql_001.py b/packstack/plugins/mysql_001.py index 203fd6d6a..bb30727c4 100644 --- a/packstack/plugins/mysql_001.py +++ b/packstack/plugins/mysql_001.py @@ -7,7 +7,7 @@ import logging from packstack.installer import validators from packstack.installer import basedefs -import packstack.installer.common_utils as utils +from packstack.installer import utils from packstack.modules.ospluginutils import getManifestTemplate, appendManifestFile @@ -16,7 +16,7 @@ controller = None # Plugin name PLUGIN_NAME = "OS-MySQL" -PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE) +PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue') logging.debug("plugin %s loaded", __name__) @@ -30,7 +30,7 @@ def initConfig(controllerObject): "PROMPT" : "Enter the IP address of the MySQL server", "OPTION_LIST" : [], "VALIDATORS" : [validators.validate_ssh], - "DEFAULT_VALUE" : utils.getLocalhostIP(), + "DEFAULT_VALUE" : utils.get_localhost_ip(), "MASK_INPUT" : False, "LOOSE_VALIDATION": True, "CONF_NAME" : "CONFIG_MYSQL_HOST", @@ -65,7 +65,7 @@ def initConfig(controllerObject): groupDict = { "GROUP_NAME" : "MYSQL", "DESCRIPTION" : "MySQL Config parameters", - "PRE_CONDITION" : utils.returnYes, + "PRE_CONDITION" : lambda x: 'yes', "PRE_CONDITION_MATCH" : "yes", "POST_CONDITION" : False, "POST_CONDITION_MATCH" : True} diff --git a/packstack/plugins/nagios_910.py b/packstack/plugins/nagios_910.py index 0299ec47d..df76f2190 100644 --- a/packstack/plugins/nagios_910.py +++ b/packstack/plugins/nagios_910.py @@ -7,7 +7,7 @@ import logging from packstack.installer import validators from packstack.installer import basedefs, output_messages -import packstack.installer.common_utils as utils +from packstack.installer import utils from packstack.modules.ospluginutils import gethostlist,\ getManifestTemplate,\ @@ -18,7 +18,7 @@ controller = None # Plugin name PLUGIN_NAME = "OS-Nagios" -PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE) +PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue') logging.debug("plugin %s loaded", __name__) @@ -32,7 +32,7 @@ def initConfig(controllerObject): "PROMPT" : "Enter the IP address of the Nagios server", "OPTION_LIST" : [], "VALIDATORS" : [validators.validate_ssh], - "DEFAULT_VALUE" : utils.getLocalhostIP(), + "DEFAULT_VALUE" : utils.get_localhost_ip(), "MASK_INPUT" : False, "LOOSE_VALIDATION": True, "CONF_NAME" : "CONFIG_NAGIOS_HOST", diff --git a/packstack/plugins/nova_300.py b/packstack/plugins/nova_300.py index b797096c5..d32f8b126 100644 --- a/packstack/plugins/nova_300.py +++ b/packstack/plugins/nova_300.py @@ -7,7 +7,7 @@ import uuid import logging from packstack.installer import validators -import packstack.installer.common_utils as utils +from packstack.installer import utils from packstack.installer.exceptions import ScriptRuntimeError from packstack.modules.ospluginutils import NovaConfig, getManifestTemplate, appendManifestFile, manifestfiles @@ -29,7 +29,7 @@ def initConfig(controllerObject): "PROMPT" : "Enter the IP address of the Nova API service", "OPTION_LIST" : [], "VALIDATORS" : [validators.validate_ip, validators.validate_ssh], - "DEFAULT_VALUE" : utils.getLocalhostIP(), + "DEFAULT_VALUE" : utils.get_localhost_ip(), "MASK_INPUT" : False, "LOOSE_VALIDATION": True, "CONF_NAME" : "CONFIG_NOVA_API_HOST", @@ -41,7 +41,7 @@ def initConfig(controllerObject): "PROMPT" : "Enter the IP address of the Nova Cert service", "OPTION_LIST" : [], "VALIDATORS" : [validators.validate_ssh], - "DEFAULT_VALUE" : utils.getLocalhostIP(), + "DEFAULT_VALUE" : utils.get_localhost_ip(), "MASK_INPUT" : False, "LOOSE_VALIDATION": True, "CONF_NAME" : "CONFIG_NOVA_CERT_HOST", @@ -53,7 +53,7 @@ def initConfig(controllerObject): "PROMPT" : "Enter the IP address of the Nova VNC proxy", "OPTION_LIST" : [], "VALIDATORS" : [validators.validate_ssh], - "DEFAULT_VALUE" : utils.getLocalhostIP(), + "DEFAULT_VALUE" : utils.get_localhost_ip(), "MASK_INPUT" : False, "LOOSE_VALIDATION": True, "CONF_NAME" : "CONFIG_NOVA_VNCPROXY_HOST", @@ -65,7 +65,7 @@ def initConfig(controllerObject): "PROMPT" : "Enter a comma separated list of IP addresses on which to install the Nova Compute services", "OPTION_LIST" : [], "VALIDATORS" : [validators.validate_multi_ssh], - "DEFAULT_VALUE" : utils.getLocalhostIP(), + "DEFAULT_VALUE" : utils.get_localhost_ip(), "MASK_INPUT" : False, "LOOSE_VALIDATION": True, "CONF_NAME" : "CONFIG_NOVA_COMPUTE_HOSTS", @@ -89,7 +89,7 @@ def initConfig(controllerObject): "PROMPT" : "Enter the IP address of the Nova Network service", "OPTION_LIST" : [], "VALIDATORS" : [validators.validate_ip, validators.validate_ssh], - "DEFAULT_VALUE" : utils.getLocalhostIP(), + "DEFAULT_VALUE" : utils.get_localhost_ip(), "MASK_INPUT" : False, "LOOSE_VALIDATION": True, "CONF_NAME" : "CONFIG_NOVA_NETWORK_HOST", @@ -101,7 +101,7 @@ def initConfig(controllerObject): "PROMPT" : "Enter the IP address of the Nova Conductor service", "OPTION_LIST" : [], "VALIDATORS" : [validators.validate_ip, validators.validate_ssh], - "DEFAULT_VALUE" : utils.getLocalhostIP(), + "DEFAULT_VALUE" : utils.get_localhost_ip(), "MASK_INPUT" : False, "LOOSE_VALIDATION": True, "CONF_NAME" : "CONFIG_NOVA_CONDUCTOR_HOST", @@ -209,7 +209,7 @@ def initConfig(controllerObject): "PROMPT" : "Enter the IP address of the Nova Scheduler service", "OPTION_LIST" : [], "VALIDATORS" : [validators.validate_ssh], - "DEFAULT_VALUE" : utils.getLocalhostIP(), + "DEFAULT_VALUE" : utils.get_localhost_ip(), "MASK_INPUT" : False, "LOOSE_VALIDATION": True, "CONF_NAME" : "CONFIG_NOVA_SCHED_HOST", diff --git a/packstack/plugins/openstack_client_400.py b/packstack/plugins/openstack_client_400.py index fc783f4f0..65ff62f21 100644 --- a/packstack/plugins/openstack_client_400.py +++ b/packstack/plugins/openstack_client_400.py @@ -6,7 +6,7 @@ import logging from packstack.installer import validators from packstack.installer import basedefs, output_messages -import packstack.installer.common_utils as utils +from packstack.installer import utils from packstack.modules.ospluginutils import getManifestTemplate, appendManifestFile @@ -15,7 +15,7 @@ controller = None # Plugin name PLUGIN_NAME = "OS-CLIENT" -PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE) +PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue') logging.debug("plugin %s loaded", __name__) @@ -29,7 +29,7 @@ def initConfig(controllerObject): "PROMPT" : "Enter the IP address of the client server", "OPTION_LIST" : [], "VALIDATORS" : [validators.validate_ssh], - "DEFAULT_VALUE" : utils.getLocalhostIP(), + "DEFAULT_VALUE" : utils.get_localhost_ip(), "MASK_INPUT" : False, "LOOSE_VALIDATION": True, "CONF_NAME" : "CONFIG_OSCLIENT_HOST", diff --git a/packstack/plugins/postscript_949.py b/packstack/plugins/postscript_949.py index 925152955..1740a2218 100644 --- a/packstack/plugins/postscript_949.py +++ b/packstack/plugins/postscript_949.py @@ -6,7 +6,7 @@ import logging from packstack.installer import validators from packstack.installer import basedefs, output_messages -import packstack.installer.common_utils as utils +from packstack.installer import utils from packstack.modules.ospluginutils import gethostlist,\ getManifestTemplate, \ @@ -28,7 +28,7 @@ def initConfig(controllerObject): groupDict = {"GROUP_NAME" : "POSTSCRIPT", "DESCRIPTION" : "POSTSCRIPT Config parameters", - "PRE_CONDITION" : utils.returnYes, + "PRE_CONDITION" : lambda x: 'yes', "PRE_CONDITION_MATCH" : "yes", "POST_CONDITION" : False, "POST_CONDITION_MATCH" : True} diff --git a/packstack/plugins/prescript_000.py b/packstack/plugins/prescript_000.py index 5b25fd576..6c4dea73f 100644 --- a/packstack/plugins/prescript_000.py +++ b/packstack/plugins/prescript_000.py @@ -6,7 +6,7 @@ import uuid import logging from packstack.installer import validators -import packstack.installer.common_utils as utils +from packstack.installer import utils from packstack.modules.ospluginutils import gethostlist,\ getManifestTemplate, \ @@ -123,7 +123,7 @@ def initConfig(controllerObject): ] groupDict = { "GROUP_NAME" : "GLOBAL", "DESCRIPTION" : "Global Options", - "PRE_CONDITION" : utils.returnYes, + "PRE_CONDITION" : lambda x: 'yes', "PRE_CONDITION_MATCH" : "yes", "POST_CONDITION" : False, "POST_CONDITION_MATCH" : True} diff --git a/packstack/plugins/puppet_950.py b/packstack/plugins/puppet_950.py index ca25811bc..2009e29ee 100644 --- a/packstack/plugins/puppet_950.py +++ b/packstack/plugins/puppet_950.py @@ -7,7 +7,7 @@ import os import platform import time -import packstack.installer.common_utils as utils +from packstack.installer import utils from packstack.installer import basedefs, output_messages from packstack.installer.exceptions import ScriptRuntimeError @@ -20,7 +20,7 @@ controller = None # Plugin name PLUGIN_NAME = "OSPUPPET" -PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE) +PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue') logging.debug("plugin %s loaded", __name__) @@ -37,7 +37,7 @@ def initConfig(controllerObject): groupDict = {"GROUP_NAME" : "PUPPET", "DESCRIPTION" : "Puppet Config parameters", - "PRE_CONDITION" : utils.returnYes, + "PRE_CONDITION" : lambda x: 'yes', "PRE_CONDITION_MATCH" : "yes", "POST_CONDITION" : False, "POST_CONDITION_MATCH" : True} @@ -162,7 +162,7 @@ def waitforpuppet(currently_running): # check the log file for errors validate_puppet_logfile(log) sys.stdout.write(("\r%s : " % log_file).ljust(basedefs.SPACE_LEN)) - print ("[ " + utils.getColoredText(output_messages.INFO_DONE, basedefs.GREEN) + " ]") + print ("[ " + utils.color_text(output_messages.INFO_DONE, 'green') + " ]") def applyPuppetManifest(): diff --git a/packstack/plugins/qpid_002.py b/packstack/plugins/qpid_002.py index 3982288c1..1ea2c04f1 100644 --- a/packstack/plugins/qpid_002.py +++ b/packstack/plugins/qpid_002.py @@ -6,7 +6,7 @@ import logging from packstack.installer import validators from packstack.installer import basedefs -import packstack.installer.common_utils as utils +from packstack.installer import utils from packstack.modules.ospluginutils import getManifestTemplate, appendManifestFile @@ -15,7 +15,7 @@ controller = None # Plugin name PLUGIN_NAME = "OS-QPID" -PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE) +PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue') logging.debug("plugin %s loaded", __name__) @@ -29,7 +29,7 @@ def initConfig(controllerObject): "PROMPT" : "Enter the IP address of the QPID service", "OPTION_LIST" : [], "VALIDATORS" : [validators.validate_ssh], - "DEFAULT_VALUE" : utils.getLocalhostIP(), + "DEFAULT_VALUE" : utils.get_localhost_ip(), "MASK_INPUT" : False, "LOOSE_VALIDATION": True, "CONF_NAME" : "CONFIG_QPID_HOST", diff --git a/packstack/plugins/serverprep_901.py b/packstack/plugins/serverprep_901.py index 286d3ca7f..9cceb859e 100644 --- a/packstack/plugins/serverprep_901.py +++ b/packstack/plugins/serverprep_901.py @@ -9,7 +9,7 @@ import datetime import platform from packstack.installer import basedefs -from packstack.installer import common_utils as utils +from packstack.installer import utils from packstack.installer import validators from packstack.installer.exceptions import InstallError @@ -21,7 +21,7 @@ controller = None # Plugin name PLUGIN_NAME = "OS-SERVERPREPARE" -PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE) +PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue') logging.debug("plugin %s loaded", __name__) @@ -241,7 +241,7 @@ def initConfig(controllerObject): conf_groups = [ {"GROUP_NAME" : "SERVERPREPARE", "DESCRIPTION" : "Server Prepare Configs ", - "PRE_CONDITION" : utils.returnYes, + "PRE_CONDITION" : lambda x: 'yes', "PRE_CONDITION_MATCH" : "yes", "POST_CONDITION" : False, "POST_CONDITION_MATCH" : True}, diff --git a/packstack/plugins/sshkeys_000.py b/packstack/plugins/sshkeys_000.py index d0c47a86f..0fab872d8 100644 --- a/packstack/plugins/sshkeys_000.py +++ b/packstack/plugins/sshkeys_000.py @@ -10,7 +10,7 @@ import tempfile from packstack.installer import processors from packstack.installer import validators from packstack.installer import basedefs -import packstack.installer.common_utils as utils +from packstack.installer import utils from packstack.modules.ospluginutils import gethostlist @@ -19,7 +19,7 @@ controller = None # Plugin name PLUGIN_NAME = "OS-SSHKEYS" -PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE) +PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue') logging.debug("plugin %s loaded", __name__) @@ -45,7 +45,7 @@ def initConfig(controllerObject): groupDict = { "GROUP_NAME" : "SSHKEY", "DESCRIPTION" : "SSH Configs ", - "PRE_CONDITION" : utils.returnYes, + "PRE_CONDITION" : lambda x: 'yes', "PRE_CONDITION_MATCH" : "yes", "POST_CONDITION" : False, "POST_CONDITION_MATCH" : True} diff --git a/packstack/plugins/swift_600.py b/packstack/plugins/swift_600.py index e06a02ffb..c4a09cb87 100644 --- a/packstack/plugins/swift_600.py +++ b/packstack/plugins/swift_600.py @@ -8,7 +8,7 @@ import os from packstack.installer import validators from packstack.installer import basedefs -import packstack.installer.common_utils as utils +from packstack.installer import utils from packstack.modules.ospluginutils import getManifestTemplate, appendManifestFile, manifestfiles @@ -17,7 +17,7 @@ controller = None # Plugin name PLUGIN_NAME = "OS-SWIFT" -PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE) +PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue') logging.debug("plugin %s loaded", __name__) @@ -31,7 +31,7 @@ def initConfig(controllerObject): "PROMPT" : "Enter the IP address of the Swift proxy service", "OPTION_LIST" : [], "VALIDATORS" : [validators.validate_ip, validators.validate_ssh], - "DEFAULT_VALUE" : utils.getLocalhostIP(), + "DEFAULT_VALUE" : utils.get_localhost_ip(), "MASK_INPUT" : False, "LOOSE_VALIDATION": True, "CONF_NAME" : "CONFIG_SWIFT_PROXY_HOSTS", #XXX: Shouldn't be here CONFIG_SWIFT_PROXY_HOST? @@ -55,7 +55,7 @@ def initConfig(controllerObject): "PROMPT" : "Enter the Swift Storage servers e.g. host/dev,host/dev", "OPTION_LIST" : [], "VALIDATORS" : [validators.validate_not_empty, validate_storage], - "DEFAULT_VALUE" : utils.getLocalhostIP(), + "DEFAULT_VALUE" : utils.get_localhost_ip(), "MASK_INPUT" : False, "LOOSE_VALIDATION": True, "CONF_NAME" : "CONFIG_SWIFT_STORAGE_HOSTS", diff --git a/tests/installer/test_utils.py b/tests/installer/test_utils.py new file mode 100644 index 000000000..a4319b181 --- /dev/null +++ b/tests/installer/test_utils.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- + +""" +Test cases for packstack.installer.utils module. +""" + +import shutil +import tempfile +from unittest import TestCase + +from ..test_base import PackstackTestCaseMixin +from packstack.installer.utils import * +from packstack.installer.utils.strings import STR_MASK +from packstack.installer.exceptions import ExecuteRuntimeError + + +cnt = 0 + + +class ParameterTestCase(PackstackTestCaseMixin, TestCase): + def setUp(self): + # Creating a temp directory that can be used by tests + self.tempdir = tempfile.mkdtemp() + + def tearDown(self): + # remove the temp directory + shutil.rmtree(self.tempdir) + + def test_sorteddict(self): + """Test packstack.installer.utils.datastructures.SortedDict""" + sdict = SortedDict() + sdict['1'] = 1 + sdict['2'] = 2 + sdict.update(SortedDict([('3', 3), ('4', 4), ('5', 5)])) + self.assertListEqual(sdict.keys(), ['1', '2', '3', '4', '5']) + self.assertListEqual(sdict.values(), [1, 2, 3, 4, 5]) + + def test_retry(self): + """Test packstack.installer.utils.decorators.retry""" + + @retry(count=3, delay=0, retry_on=ValueError) + def test_sum(): + global cnt + cnt += 1 + raise ValueError + + global cnt + cnt = 0 + + try: + test_sum() + except ValueError: + pass + self.assertEqual(cnt, 4) + self.assertRaises(ValueError, test_sum) + + def test_network(self): + """Test packstack.installer.utils.network functions""" + self.assertIn(host2ip('google-public-dns-a.google.com'), + ['8.8.8.8', '2001:4860:4860::8888']) + self.assertIn(host2ip('localhost', allow_localhost=True), + ['127.0.0.1', '::1']) + + def test_shell(self): + """Test packstack.installer.utils.shell functions""" + rc, out = execute(['echo', 'this is test']) + self.assertEqual(out.strip(), 'this is test') + rc, out = execute('echo "this is test"', use_shell=True) + self.assertEqual(out.strip(), 'this is test') + try: + execute('echo "mask the password" && exit 1', + use_shell=True, mask_list=['password']) + raise AssertionError('Masked execution failed.') + except ExecuteRuntimeError, ex: + should_be = ('Failed to execute command: ' + 'echo "mask the %s" && exit 1' % STR_MASK) + self.assertEqual(str(ex), should_be) + + script = ScriptRunner() + script.append('echo "this is test"') + rc, out = script.execute() + self.assertEqual(out.strip(), 'this is test') + + def test_strings(self): + """Test packstack.installer.utils.strings functions""" + self.assertEqual(color_text('test text', 'red'), + '\033[0;31mtest text\033[0m') + self.assertEqual(mask_string('test text', mask_list=['text']), + 'test %s' % STR_MASK) + masked = mask_string("test '\\''text'\\''", + mask_list=["'text'"], + replace_list=[("'", "'\\''")]) + self.assertEqual(masked, 'test %s' % STR_MASK)