Cleaned packstack.installer.common_utils

- cleaned from useless functions
- refactored to be pep8 compliant
- separated to standalone modules
- added unit tests

Change-Id: If5a300fe04efab9e13549f2b4d05c1b28e2e40c6

packstack/installer/basedefs.py

@@ -1,11 +1,15 @@
+# -*- coding: utf-8 -*-
+
 """
-provides all the predefined variables for engine-setup
+This module provides all the predefined variables.
 """
+
 import os
 import sys
 import datetime
 import tempfile
 
+
 APP_NAME = "Installer"
 
 FILE_YUM_VERSION_LOCK = "/etc/yum/pluginconf.d/versionlock.list"
@@ -17,7 +21,7 @@ except:
     pass
 
 _tmpdirprefix = datetime.datetime.now().strftime('%Y%m%d-%H%M%S-')
-VAR_DIR = tempfile.mkdtemp(prefix = _tmpdirprefix, dir = PACKSTACK_VAR_DIR)
+VAR_DIR = tempfile.mkdtemp(prefix=_tmpdirprefix, dir=PACKSTACK_VAR_DIR)
 DIR_LOG = VAR_DIR
 PUPPET_MANIFEST_RELATIVE = "manifests"
 PUPPET_MANIFEST_DIR = os.path.join(VAR_DIR, PUPPET_MANIFEST_RELATIVE)
@@ -37,17 +41,10 @@ EXEC_CHKCONFIG = "chkconfig"
 EXEC_SERVICE = "service"
 EXEC_IP = "ip"
 
-#text colors
+# text colors
-RED = "\033[0;31m"
-GREEN = "\033[92m"
-BLUE = "\033[94m"
-YELLOW = "\033[93m"
 NO_COLOR = "\033[0m"
+COLORS = {'red': "\033[0;31m", 'green': "\033[92m", 'blue': "\033[94m",
+          'yellow': "\033[93m"}
 
-COLORS = (RED, GREEN, BLUE, YELLOW, NO_COLOR)
+# space len size for color print
-
-#space len size for color print
 SPACE_LEN = 70
-
-RPM_LOCK_LIST = """
-"""

packstack/installer/common_utils.py

@@ -1,431 +0,0 @@
-# -*- coding: utf-8 -*-
-
-"""
-contains all common and re-usable code for rhevm-setup and sub packages
-"""
-import grp
-import pwd
-import logging
-import subprocess
-import re
-import traceback
-import os
-import datetime
-import types
-import time
-import socket
-import tempfile
-
-import basedefs
-import output_messages
-from .exceptions import NetworkError, ScriptRuntimeError
-
-
-
-def getColoredText (text, color):
-    ''' gets text string and color
-        and returns a colored text.
-        the color values are RED/BLUE/GREEN/YELLOW
-        everytime we color a text, we need to disable
-        the color at the end of it, for that
-        we use the NO_COLOR chars.
-    '''
-    return color + text + basedefs.NO_COLOR
-
-def execCmd(cmdList, cwd=None, failOnError=False, msg=output_messages.ERR_RC_CODE, maskList=[], useShell=False, usePipeFiles=False):
-    """
-    Run external shell command with 'shell=false'
-    receives a list of arguments for command line execution
-    """
-    # All items in the list needs to be strings, otherwise the subprocess will fail
-    cmd = [str(item) for item in cmdList]
-
-    # We need to join cmd list into one string so we can look for passwords in it and mask them
-    logCmd = _maskString((' '.join(cmd)), maskList)
-    logging.debug("Executing command --> '%s'"%(logCmd))
-
-    stdErrFD = subprocess.PIPE
-    stdOutFD = subprocess.PIPE
-    stdInFD = subprocess.PIPE
-
-    if usePipeFiles:
-        (stdErrFD, stdErrFile) = tempfile.mkstemp(dir="/tmp")
-        (stdOutFD, stdOutFile) = tempfile.mkstemp(dir="/tmp")
-        (stdInFD, stdInFile) = tempfile.mkstemp(dir="/tmp")
-
-    # We use close_fds to close any file descriptors we have so it won't be copied to forked childs
-    proc = subprocess.Popen(cmd, stdout=stdOutFD,
-            stderr=stdErrFD, stdin=stdInFD, cwd=cwd, shell=useShell, close_fds=True)
-
-    out, err = proc.communicate()
-    if usePipeFiles:
-        with open(stdErrFile, 'r') as f:
-            err = f.read()
-        os.remove(stdErrFile)
-
-        with open(stdOutFile, 'r') as f:
-            out = f.read()
-        os.remove(stdOutFile)
-        os.remove(stdInFile)
-
-    logging.debug("output = %s"%(out))
-    logging.debug("stderr = %s"%(err))
-    logging.debug("retcode = %s"%(proc.returncode))
-    output = out + err
-    if failOnError and proc.returncode != 0:
-        raise Exception(msg)
-    return ("".join(output.splitlines(True)), proc.returncode)
-
-def byLength(word1, word2):
-    """
-    Compars two strings by their length
-    Returns:
-    Negative if word2 > word1
-    Positive if word1 > word2
-    Zero if word1 == word 2
-    """
-    return len(word1) - len(word2)
-
-def nslookup(address):
-    cmd = [
-        basedefs.EXEC_NSLOOKUP, address,
-    ]
-    #since nslookup will return 0 no matter what, the RC is irrelevant
-    output, rc = execCmd(cmdList=cmd)
-    return output
-
-def getConfiguredIps():
-    try:
-        iplist=set()
-        cmd = [
-            basedefs.EXEC_IP, "addr",
-        ]
-        output, rc = execCmd(cmdList=cmd, failOnError=True, msg=output_messages.ERR_EXP_GET_CFG_IPS_CODES)
-        ipaddrPattern=re.compile('\s+inet (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}).+')
-        list=output.splitlines()
-        for line in list:
-            foundIp = ipaddrPattern.search(line)
-            if foundIp:
-                if foundIp.group(1) != "127.0.0.1":
-                    ipAddr = foundIp.group(1)
-                    logging.debug("Found IP Address: %s"%(ipAddr))
-                    iplist.add(ipAddr)
-        return iplist
-    except:
-        logging.error(traceback.format_exc())
-        raise Exception(output_messages.ERR_EXP_GET_CFG_IPS)
-
-def getCurrentDateTime(isUtc=None):
-    now = None
-    if (isUtc is not None):
-        now = datetime.datetime.utcnow()
-    else:
-        now = datetime.datetime.now()
-    return now.strftime("%Y_%m_%d_%H_%M_%S")
-
-def verifyStringFormat(str, matchRegex):
-    '''
-    Verify that the string given matches the matchRegex.
-    for example:
-    string: 111-222
-    matchRegex: \d{3}-\d{3}
-    this will return true since the string matches the regex
-    '''
-    pattern = re.compile(matchRegex)
-    result = re.match(pattern, str)
-    if result == None:
-        return False
-    else:
-        return True
-
-def compareStrIgnoreCase(str1, str2):
-    ''' compare 2 strings and ignore case
-        if one of the input is not str (bool for e.g) - return normal comapre
-    '''
-    if type(str1) == types.StringType and type(str2) == types.StringType:
-        return str1.lower() == str2.lower()
-    else:
-        return str1 == str2
-
-def parseStrRegex(string, regex, errMsg):
-    """
-    Gets a text string and a regex pattern
-    and returns the extracted sub-string
-    captured.
-    """
-    rePattern = re.compile(regex)
-    found = rePattern.search(string)
-    if found:
-        match = found.group(1)
-        logging.debug("found new parsed string: %s"%(match))
-        return match
-    else:
-        raise Exception(errMsg)
-
-def _maskString(string, maskList=[]):
-    """
-    private func to mask passwords
-    in utils
-    """
-    maskedStr = string
-    for maskItem in maskList:
-        if not maskItem: continue
-        maskedStr = maskedStr.replace(maskItem, "*"*8)
-        # if looking at stderr of a script, single quotes have been converted
-        # to '\''
-        maskedStr = maskedStr.replace(maskItem.replace("'","'\\''"), "*"*8)
-
-    return maskedStr
-
-def retry(func, expectedException=Exception, tries=None, timeout=None, sleep=1):
-    """
-    Retry a function. Wraps the retry logic so you don't have to
-    implement it each time you need it.
-
-    :param func: The callable to run.
-    :param expectedException: The exception you expect to receive when the function fails.
-    :param tries: The number of time to try. None\0,-1 means infinite.
-    :param timeout: The time you want to spend waiting. This **WILL NOT** stop the method.
-                    It will just not run it if it ended after the timeout.
-    :param sleep: Time to sleep between calls in seconds.
-    """
-    if tries in [0, None]:
-        tries = -1
-
-    if timeout in [0, None]:
-        timeout = -1
-
-    startTime = time.time()
-
-    while True:
-        tries -= 1
-        try:
-            return func()
-        except expectedException:
-            if tries == 0:
-                raise
-
-            if (timeout > 0) and ((time.time() - startTime) > timeout):
-                raise
-
-            time.sleep(sleep)
-
-def localHost(hostname):
-    # Create an ip set of possible IPs on the machine. Set has only unique values, so
-    # there's no problem with union.
-    # TODO: cache the list somehow? There's no poing quering the IP configuraion all the time.
-    ipset = getConfiguredIps().union(set([ "localhost", "127.0.0.1"]))
-    if hostname in ipset:
-        return True
-    return False
-
-# TODO: Support SystemD services
-class Service():
-    def __init__(self, name):
-        self.wasStopped = False
-        self.wasStarted = False
-        self.name = name
-
-    def isServiceAvailable(self):
-        if os.path.exists("/etc/init.d/%s" % self.name):
-            return True
-        return False
-
-    def start(self, raiseFailure = False):
-        logging.debug("starting %s", self.name)
-        (output, rc) = self._serviceFacility("start")
-        if rc == 0:
-            self.wasStarted = True
-        elif raiseFailure:
-            raise Exception(output_messages.ERR_FAILED_START_SERVICE % self.name)
-
-        return (output, rc)
-
-    def stop(self, raiseFailure = False):
-        logging.debug("stopping %s", self.name)
-        (output, rc) = self._serviceFacility("stop")
-        if rc == 0:
-            self.wasStopped = True
-        elif raiseFailure:
-                raise Exception(output_messages.ERR_FAILED_STOP_SERVICE % self.name)
-
-        return (output, rc)
-
-    def autoStart(self, start=True):
-        mode = "on" if start else "off"
-        cmd = [
-            basedefs.EXEC_CHKCONFIG, self.name, mode,
-        ]
-        execCmd(cmdList=cmd, failOnError=True)
-
-    def conditionalStart(self, raiseFailure = False):
-        """
-        Will only start if wasStopped is set to True
-        """
-        if self.wasStopped:
-            logging.debug("Service %s was stopped. starting it again"%self.name)
-            return self.start(raiseFailure)
-        else:
-            logging.debug("Service was not stopped. there for we're not starting it")
-            return (False, False)
-
-    def status(self):
-        logging.debug("getting status for %s", self.name)
-        (output, rc) = self._serviceFacility("status")
-        return (output, rc)
-
-    def _serviceFacility(self, action):
-        """
-        Execute the command "service NAME action"
-        returns: output, rc
-        """
-        logging.debug("executing action %s on service %s", self.name, action)
-        cmd = [
-            basedefs.EXEC_SERVICE, self.name, action
-        ]
-        return execCmd(cmdList=cmd, usePipeFiles=True)
-
-def chown(target,uid, gid):
-    logging.debug("chown %s to %s:%s" % (target, uid, gid))
-    os.chown(target, uid, gid)
-
-def installed(rpm):
-    cmd = [
-        basedefs.EXEC_RPM,
-        "-q",
-        rpm,
-    ]
-    output, rc = execCmd(cmd)
-    return rc == 0
-
-def returnYes(controller):
-    return "yes"
-
-def getLocalhostIP():
-    """
-    Returns IP address of localhost.
-    """
-    # TO-DO: Will probably need to find better way to find out localhost
-    #        address.
-
-    # find nameservers
-    ns_regex = re.compile('nameserver\s*(?P<ns_ip>[\d\.\:])')
-    resolv, rc = execCmd(['cat /etc/resolv.conf | grep nameserver',],
-                         failOnError=False, useShell=True)
-    nsrvs = []
-    for line in resolv.split('\n'):
-        match = ns_regex.match(line.strip())
-        if match:
-            nsrvs.append(match.group('ns_ip'))
-
-    # try to connect to nameservers and return own IP address
-    nsrvs.append('8.8.8.8') # default to google dns
-    for i in nsrvs:
-        try:
-            s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-            s.connect((i, 0))
-            loc_ip = s.getsockname()[0]
-        except socket.error:
-            continue
-        else:
-            return loc_ip
-
-def host2ip(hostname, allow_localhost=False):
-    """
-    Converts given hostname to IP address. Raises NetworkError
-    if conversion failed.
-    """
-    try:
-        ip_list = socket.gethostbyaddr(hostname)[2]
-        if allow_localhost:
-            return ip_list[0]
-        else:
-            local_ips = ('127.0.0.1', '::1')
-            for ip in ip_list:
-                if ip not in local_ips:
-                    break
-            else:
-                raise NameError()
-            return ip
-    except NameError:
-        # given hostname is localhost, return appropriate IP address
-        ip = getLocalhostIP()
-        if not ip:
-            raise NetworkError('Failed to get local IP address.')
-        return ip
-    except socket.error:
-        raise NetworkError('Unknown hostname %s.' % hostname)
-    except Exception, ex:
-        raise NetworkError('Unknown error appeared: %s' % repr(ex))
-
-def forceIP(host, allow_localhost=False):
-    host = host.strip()
-    ipv4_regex = re.compile('\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}')
-    ipv6_regex = re.compile('[abcdef\d\:]+')
-    if not ipv4_regex.match(host) or not ipv6_regex.match(host):
-        host = host2ip(host, allow_localhost=allow_localhost)
-    return host
-
-def device_from_ip(ip):
-    server = ScriptRunner()
-    server.append("DEVICE=$(ip address show to %s | head -n 1 | sed -e 's/.*: \(.*\):.*/\\1/g')" % ip)
-    # Test device, raises an exception is it doesn't exist
-    server.append("ip link show \"$DEVICE\" > /dev/null")
-    server.append("echo $DEVICE")
-    rv, stdout = server.execute()
-    return stdout.strip()
-
-class ScriptRunner(object):
-    def __init__(self, ip=None):
-        self.script = []
-        self.ip = ip
-
-    def append(self, s):
-        self.script.append(s)
-
-    def clear(self):
-        self.script = []
-
-    def execute(self, logerrors=True, maskList=None):
-        maskList = maskList or []
-        script = "\n".join(self.script)
-        logging.debug("# ============ ssh : %r =========="%self.ip)
-
-        _PIPE = subprocess.PIPE  # pylint: disable=E1101
-        if self.ip:
-            cmd = ["ssh", "-o", "StrictHostKeyChecking=no",
-                          "-o", "UserKnownHostsFile=/dev/null",
-                          "root@%s" % self.ip, "bash -x"]
-        else:
-            cmd = ["bash", "-x"]
-        obj = subprocess.Popen(cmd, stdin=_PIPE, stdout=_PIPE, stderr=_PIPE,
-                               close_fds=True, shell=False)
-
-        logging.debug(_maskString(script, maskList))
-        script = "function t(){ exit $? ; } \n trap t ERR \n" + script
-        stdoutdata, stderrdata = obj.communicate(script)
-        logging.debug("============ STDOUT ==========")
-        logging.debug(_maskString(stdoutdata, maskList))
-        returncode = obj.returncode
-        if returncode:
-            if logerrors:
-                logging.error("============= STDERR ==========")
-                logging.error(_maskString(stderrdata, maskList))
-
-            pattern = (r'^ssh\:')
-            if re.search(pattern, stderrdata):
-                raise NetworkError(stderrdata)
-            else:
-                raise ScriptRuntimeError('Error running remote script: '
-                                         '%s' % stdoutdata)
-        return returncode, stdoutdata
-
-    def template(self, src, dst, varsdict):
-        with open(src) as fp:
-            self.append("cat > %s <<- EOF\n%s\nEOF\n"%(dst, fp.read()%varsdict))
-
-    def ifnotexists(self, fn, s):
-        self.append("[ -e %s ] || %s"%(fn, s))
-
-    def ifexists(self, fn, s):
-        self.append("[ -e %s ] && %s"%(fn, s))

packstack/installer/exceptions.py

@@ -18,7 +18,11 @@ __all__ = (
 
 class PackStackError(Exception):
     """Default Exception class for packstack installer."""
-    pass
+    def __init__(self, *args, **kwargs):
+        super(PackStackError, self).__init__(*args)
+        self.stdout = kwargs.get('stdout', None)
+        self.stderr = kwargs.get('stderr', None)
+
 
 
 class MissingRequirements(PackStackError):
@@ -55,5 +59,10 @@ class NetworkError(PackStackError):
 
 
 class ScriptRuntimeError(PackStackError):
-    """Raised when ScriptRunner.execute does not end successfully."""
+    """
+    Raised when utils.ScriptRunner.execute does not end successfully.
+    """
     pass
+
+class ExecuteRuntimeError(PackStackError):
+    """Raised when utils.execute does not end successfully."""

packstack/installer/output_messages.py

@@ -77,7 +77,6 @@ ERR_IPS_NOT_CONFIGED_ON_INT="The IP (%s) which was resolved from the FQDN %s is
 ERR_IPS_HAS_NO_PTR="None of the IP addresses on this host(%s) holds a PTR record for the FQDN: %s"
 ERR_IP_HAS_NO_PTR="The IP %s does not hold a PTR record for the FQDN: %s"
 ERR_EXP_FAILED_INIT_LOGGER="Unexpected error: Failed to initiate logger, please check file system permission"
-ERR_RC_CODE="Return Code is not zero"
 ERR_FAILURE="General failure"
 ERR_NO_ANSWER_FILE="Error: Could not find file %s"
 ERR_ONLY_1_FLAG="Error: The %s flag is mutually exclusive to all other command line options"

packstack/installer/processors.py

@@ -2,7 +2,7 @@
 
 import os
 
-from .common_utils import ScriptRunner, forceIP
+from .utils import ScriptRunner, force_ip
 from .exceptions import ParamProcessingError, NetworkError
 
 
@@ -17,7 +17,7 @@ def process_host(param, process_args=None):
     localhost = process_args and \
                 process_args.get('allow_localhost', False)
     try:
-        return forceIP(param, allow_localhost=localhost)
+        return force_ip(param, allow_localhost=localhost)
     except NetworkError, ex:
         raise ParamProcessingError(str(ex))
 

packstack/installer/run_setup.py

@@ -15,7 +15,7 @@ from optparse import OptionParser, OptionGroup
 
 import basedefs
 import validators
-import common_utils as utils
+from . import utils
 import processors
 import output_messages
 from .exceptions import FlagValidationError, ParamValidationError
@@ -35,7 +35,6 @@ def initLogging (debug):
     global logFile
 
     try:
-        #in order to use UTC date for the log file, send True to getCurrentDateTime(True)
         logFilename = "openstack-setup.log"
         logFile = os.path.join(basedefs.DIR_LOG, logFilename)
 
@@ -220,18 +219,19 @@ def mask(input):
     if type(input) == types.DictType:
         for key in input:
             if type(input[key]) == types.StringType:
-                output[key] = maskString(input[key])
+                output[key] = utils.mask_string(input[key],
+                                                masked_value_set)
     if type(input) == types.ListType:
         for item in input:
             org = item
             orgIndex = input.index(org)
             if type(item) == types.StringType:
-                item = maskString(item)
+                item = utils.mask_string(item, masked_value_set)
             if item != org:
                 output.remove(org)
                 output.insert(orgIndex, item)
     if type(input) == types.StringType:
-            output = maskString(input)
+            output = utils.mask_string(input, masked_value_set)
 
     return output
 
@@ -253,13 +253,6 @@ def removeMaskString(maskedString):
     if found:
         masked_value_set.remove(maskedString)
 
-def maskString(str):
-    # Iterate sorted list, so we won't mask only part of a password
-    for password in sorted(masked_value_set, utils.byLength, None, True):
-        if password:
-            str = str.replace(password, '*'*8)
-    return str
-
 def validate_param_value(param, value):
     cname = param.CONF_NAME
     logging.debug("Validating parameter %s." % cname)
@@ -364,7 +357,7 @@ def _handleAnswerFileParams(answerFile):
 
             # Handle pre condition match with case insensitive values
             logging.info("Comparing pre- conditions, value: '%s', and match: '%s'" % (preConditionValue, group.PRE_CONDITION_MATCH))
-            if utils.compareStrIgnoreCase(preConditionValue, group.PRE_CONDITION_MATCH):
+            if preConditionValue == group.PRE_CONDITION_MATCH:
                 for param in group.parameters.itervalues():
                     _loadParamFromFile(fconf, "general", param.CONF_NAME)
 
@@ -374,7 +367,7 @@ def _handleAnswerFileParams(answerFile):
                     postConditionValue = _handleGroupCondition(fconf, group.POST_CONDITION, postConditionValue)
 
                     # Handle post condition match for group
-                    if not utils.compareStrIgnoreCase(postConditionValue, group.POST_CONDITION_MATCH):
+                    if postConditionValue != group.POST_CONDITION_MATCH:
                         logging.error("The group condition (%s) returned: %s, which differs from the excpeted output: %s"%\
                                       (group.GROUP_NAME, postConditionValue, group.POST_CONDITION_MATCH))
                         raise ValueError(output_messages.ERR_EXP_GROUP_VALIDATION_ANS_FILE%\
@@ -422,7 +415,7 @@ def _handleInteractiveParams():
 
             # If we have a match, i.e. condition returned True, go over all params in the group
             logging.info("Comparing pre-conditions; condition: '%s', and match: '%s'" % (preConditionValue, group.PRE_CONDITION_MATCH))
-            if utils.compareStrIgnoreCase(preConditionValue, group.PRE_CONDITION_MATCH):
+            if preConditionValue == group.PRE_CONDITION_MATCH:
                 while inputLoop:
                     for param in group.parameters.itervalues():
                         if not param.CONDITION:
@@ -546,19 +539,6 @@ def _addFinalInfoMsg():
     """
     controller.MESSAGES.append(output_messages.INFO_LOG_FILE_PATH%(logFile))
 
-def _lockRpmVersion():
-    """
-    Enters rpm versions into yum version-lock
-    """
-    logging.debug("Locking rpms in yum-version-lock")
-    cmd = [
-        basedefs.EXEC_RPM, "-q",
-    ] + basedefs.RPM_LOCK_LIST.split()
-    output, rc = utils.execCmd(cmdList=cmd, failOnError=True, msg=output_messages.ERR_YUM_LOCK)
-
-    with open(basedefs.FILE_YUM_VERSION_LOCK, "a") as f:
-        for rpm in output.splitlines():
-            f.write(rpm + "\n")
 
 def _summaryParamsToLog():
     if len(controller.CONF) > 0:
@@ -624,7 +604,7 @@ def remove_remote_var_dirs():
             msg = output_messages.ERR_REMOVE_REMOTE_VAR % (host_dir, host)
             logging.error(msg)
             logging.exception(e)
-            controller.MESSAGES.append(utils.getColoredText(msg, basedefs.RED))
+            controller.MESSAGES.append(utils.color_text(msg, 'red'))
 
 
 def generateAnswerFile(outputFile, overrides={}):
@@ -905,7 +885,7 @@ def main():
     except Exception as e:
         logging.error(traceback.format_exc())
         print
-        print utils.getColoredText("ERROR : "+str(e), basedefs.RED)
+        print utils.color_text("ERROR : " + str(e), 'red')
         print output_messages.ERR_CHECK_LOG_FILE_FOR_MORE_INFO%(logFile)
         sys.exit(1)
 

packstack/installer/setup_params.py

@@ -4,7 +4,7 @@
 Container set for groups and parameters
 """
 
-from .datastructures import SortedDict
+from .utils.datastructures import SortedDict
 
 
 class Parameter(object):

packstack/installer/setup_sequences.py

@@ -8,7 +8,7 @@ import string
 import traceback
 import basedefs
 import output_messages
-import common_utils as utils
+from . import utils
 
 class Step(object):
     def __init__(self, title=None, functions=[]):
@@ -61,7 +61,7 @@ class Step(object):
             except:
                 logging.debug(traceback.format_exc())
                 raise
-        print ("[ " + utils.getColoredText(output_messages.INFO_DONE, basedefs.GREEN) + " ]").rjust(spaceLen)
+        print ("[ " + utils.color_text(output_messages.INFO_DONE, 'green') + " ]").rjust(spaceLen)
 
 class Sequence(object):
     """

packstack/installer/utils/__init__.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+
+from .datastructures import SortedDict
+from .decorators import retry
+from .network import get_localhost_ip, host2ip, force_ip
+from .shell import ScriptRunner, execute
+from .strings import color_text, mask_string
+
+
+__all__ = ('SortedDict',
+           'retry',
+           'ScriptRunner', 'execute',
+           'get_localhost_ip', 'host2ip', 'force_ip',
+           'color_text', 'mask_string')

packstack/installer/utils/decorators.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+
+import time
+
+
+def retry(count=1, delay=0, retry_on=Exception):
+    """
+    Decorator which tries to run specified fuction if the previous
+    run ended by given exception. Retry count and delays can be also
+    specified.
+    """
+    if count < 0 or delay < 0:
+        raise ValueError('Count and delay has to be positive number.')
+
+    def decorator(func):
+        def wrapper(*args, **kwargs):
+            tried = 0
+            while tried <= count:
+                try:
+                    return func(*args, **kwargs)
+                except retry_on:
+                    if tried >= count:
+                        raise
+                    if delay:
+                        time.sleep(delay)
+                    tried += 1
+        wrapper.func_name = func.func_name
+        return wrapper
+    return decorator

packstack/installer/utils/network.py

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+
+import re
+import socket
+
+from .shell import execute
+
+
+def get_localhost_ip():
+    """
+    Returns IP address of localhost.
+    """
+    # TO-DO: Will probably need to find better way to find out localhost
+    #        address.
+
+    # find nameservers
+    ns_regex = re.compile('nameserver\s*(?P<ns_ip>[\d\.\:])')
+    rc, resolv = execute('cat /etc/resolv.conf | grep nameserver',
+                         can_fail=False, use_shell=True)
+    nsrvs = []
+    for line in resolv.split('\n'):
+        match = ns_regex.match(line.strip())
+        if match:
+            nsrvs.append(match.group('ns_ip'))
+
+    # try to connect to nameservers and return own IP address
+    nsrvs.append('8.8.8.8') # default to google dns
+    for i in nsrvs:
+        try:
+            s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+            s.connect((i, 0))
+            loc_ip = s.getsockname()[0]
+        except socket.error:
+            continue
+        else:
+            return loc_ip
+
+
+def host2ip(hostname, allow_localhost=False):
+    """
+    Converts given hostname to IP address. Raises NetworkError
+    if conversion failed.
+    """
+    try:
+        ip_list = socket.gethostbyaddr(hostname)[2]
+        if allow_localhost:
+            return ip_list[0]
+        else:
+            local_ips = ('127.0.0.1', '::1')
+            for ip in ip_list:
+                if ip not in local_ips:
+                    break
+            else:
+                raise NameError()
+            return ip
+    except NameError:
+        # given hostname is localhost, return appropriate IP address
+        ip = get_localhost_ip()
+        if not ip:
+            raise NetworkError('Failed to get local IP address.')
+        return ip
+    except socket.error:
+        raise NetworkError('Unknown hostname %s.' % hostname)
+    except Exception, ex:
+        raise NetworkError('Unknown error appeared: %s' % repr(ex))
+
+
+def force_ip(host, allow_localhost=False):
+    host = host.strip()
+    ipv4_regex = re.compile('\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}')
+    ipv6_regex = re.compile('[abcdef\d\:]+')
+    if not ipv4_regex.match(host) or not ipv6_regex.match(host):
+        host = host2ip(host, allow_localhost=allow_localhost)
+    return host

packstack/installer/utils/shell.py

@@ -0,0 +1,118 @@
+# -*- coding: utf-8 -*-
+
+import re
+import types
+import logging
+import subprocess
+
+from ..exceptions import (ExecuteRuntimeError, ScriptRuntimeError,
+                          NetworkError)
+from .strings import mask_string
+
+
+def execute(cmd, workdir=None, can_fail=False, mask_list=None,
+            use_shell=False):
+    """
+    Runs shell command cmd. If can_fail is set to False
+    ExecuteRuntimeError is raised if command returned non-zero return
+    code. Otherwise
+    """
+    mask_list = mask_list or []
+    repl_list = [("'","'\\''")]
+
+    if type(cmd) is not types.StringType:
+        import pipes
+        masked = ' '.join((pipes.quote(i) for i in cmd))
+    else:
+        masked = cmd
+    masked = mask_string(masked, mask_list, repl_list)
+    logging.debug("Executing command: %s" % masked)
+
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                            stderr=subprocess.PIPE, cwd=workdir,
+                            shell=use_shell, close_fds=True)
+    out, err = proc.communicate()
+    logging.debug("rc: %s" % proc.returncode)
+    logging.debug("stdout:\n%s" % mask_string(out, mask_list, repl_list))
+    logging.debug("stderr:\n%s" % mask_string(err, mask_list, repl_list))
+
+    if not can_fail and proc.returncode != 0:
+        msg = 'Failed to execute command: %s' % masked
+        raise ExecuteRuntimeError(msg, stdout=out, stderr=err)
+    return proc.returncode, out
+
+
+class ScriptRunner(object):
+    _pkg_search = 'rpm -q'
+
+    def __init__(self, ip=None):
+        self.script = []
+        self.ip = ip
+
+    def append(self, s):
+        self.script.append(s)
+
+    def clear(self):
+        self.script = []
+
+    def execute(self, logerrors=True, maskList=None):
+        maskList = maskList or []
+        script = "\n".join(self.script)
+        logging.debug("# ============ ssh : %r =========="%self.ip)
+
+        _PIPE = subprocess.PIPE  # pylint: disable=E1101
+        if self.ip:
+            cmd = ["ssh", "-o", "StrictHostKeyChecking=no",
+                          "-o", "UserKnownHostsFile=/dev/null",
+                          "root@%s" % self.ip, "bash -x"]
+        else:
+            cmd = ["bash", "-x"]
+        obj = subprocess.Popen(cmd, stdin=_PIPE, stdout=_PIPE,
+                               stderr=_PIPE, close_fds=True,
+                               shell=False)
+
+        replace_list = [("'","'\\''")]
+        logging.debug(mask_string(script, maskList, replace_list))
+        script = "function t(){ exit $? ; } \n trap t ERR \n" + script
+        stdoutdata, stderrdata = obj.communicate(script)
+        logging.debug("============ STDOUT ==========")
+        logging.debug(mask_string(stdoutdata, maskList, replace_list))
+        returncode = obj.returncode
+        if returncode:
+            if logerrors:
+                logging.error("============= STDERR ==========")
+                logging.error(mask_string(stderrdata, maskList,
+                                          replace_list))
+
+            pattern = (r'^ssh\:')
+            if re.search(pattern, stderrdata):
+                raise NetworkError(stderrdata, stdout=stdoutdata,
+                                   stderr=stderrdata)
+            else:
+                msg = 'Error running remote script: %s' % stdoutdata
+                raise ScriptRuntimeError(msg, stdout=stdoutdata,
+                                         stderr=stderrdata)
+        return returncode, stdoutdata
+
+    def template(self, src, dst, varsdict):
+        with open(src) as fp:
+            content = fp.read() % varsdict
+            self.append("cat > %s <<- EOF\n%s\nEOF\n" % (dst, content))
+
+    def if_not_exists(self, path, command):
+        self.append("[ -e %s ] || %s" % (path, command))
+
+    def if_exists(self, path, command):
+        self.append("[ -e %s ] && %s" % (path, command))
+
+    def if_installed(self, pkg, command):
+        self.append("%s %s && %s" % (self._pkg_search, pkg, command))
+
+    def if_not_installed(self, pkg, command):
+        self.append("%s %s || %s" % (self._pkg_search, pkg, command))
+
+    def chown(self, target, uid, gid):
+        self.append("chown %s:%s %s" % (uid, gid, target))
+
+    def chmod(self, target, mode):
+        self.append("chown %s %s" % (mode, target))

packstack/installer/utils/strings.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+
+from .. import basedefs
+
+
+STR_MASK = '*' * 8
+
+
+def color_text(text, color):
+    """
+    Returns given text string with appropriate color tag. Allowed values
+    for color parameter are 'red', 'blue', 'green' and 'yellow'.
+    """
+    return '%s%s%s' % (basedefs.COLORS[color], text, basedefs.NO_COLOR)
+
+
+def mask_string(unmasked, mask_list=None, replace_list=None):
+    """
+    Replaces words from mask_list with MASK in unmasked string.
+    If words are needed to be transformed before masking, transformation
+    could be describe in replace list. For example [("'","'\\''")]
+    replaces all ' characters with '\\''.
+    """
+    mask_list = mask_list or []
+    replace_list = replace_list or []
+
+    masked = unmasked
+    for word in sorted(mask_list, lambda x, y: len(y) - len(x)):
+        if not word:
+            continue
+        for before, after in replace_list:
+            word = word.replace(before, after)
+        masked = masked.replace(word, STR_MASK)
+    return masked

packstack/installer/validators.py

@@ -12,7 +12,7 @@ import tempfile
 import traceback
 
 import basedefs
-import common_utils as utils
+from . import utils
 
 from .setup_controller import Controller
 from .exceptions import ParamValidationError
@@ -175,8 +175,8 @@ def validate_ping(param, options=None):
     # TO-DO: to be more flexible, remove this and exit in case param is empty
     validate_not_empty(param)
 
-    cmd = ["/bin/ping", "-c", "1", str(param)]
+    rc, out = utils.execute(['/bin/ping', '-c', '1', str(param)],
-    out, rc = utils.execCmd(cmdList=cmd)
+                            can_fail=True)
     if rc != 0:
         logging.debug('validate_ping(%s, options=%s) failed.' %
                       (param, options))

packstack/plugins/cinder_250.py

@@ -10,7 +10,7 @@ from packstack.installer import exceptions
 from packstack.installer import validators
 
 from packstack.installer import basedefs
-import packstack.installer.common_utils as utils
+from packstack.installer import utils
 
 from packstack.modules.ospluginutils import getManifestTemplate, appendManifestFile
 from packstack.installer.exceptions import ScriptRuntimeError
@@ -22,7 +22,7 @@ controller = None
 
 # Plugin name
 PLUGIN_NAME = "OS-Cinder"
-PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE)
+PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue')
 
 logging.debug("plugin %s loaded", __name__)
 
@@ -37,7 +37,7 @@ def initConfig(controllerObject):
                    "PROMPT"          : "Enter the IP address of the Cinder server",
                    "OPTION_LIST"     : [],
                    "VALIDATORS"      : [validators.validate_ssh],
-                   "DEFAULT_VALUE"   : utils.getLocalhostIP(),
+                   "DEFAULT_VALUE"   : utils.get_localhost_ip(),
                    "MASK_INPUT"      : False,
                    "LOOSE_VALIDATION": True,
                    "CONF_NAME"       : "CONFIG_CINDER_HOST",

packstack/plugins/dashboard_500.py

@@ -9,7 +9,7 @@ import uuid
 from packstack.installer import validators
 from packstack.installer import basedefs, output_messages
 from packstack.installer import exceptions
-import packstack.installer.common_utils as utils
+from packstack.installer import utils
 
 from packstack.modules.ospluginutils import getManifestTemplate, appendManifestFile
 
@@ -18,7 +18,7 @@ controller = None
 
 # Plugin name
 PLUGIN_NAME = "OS-HORIZON"
-PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE)
+PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue')
 
 logging.debug("plugin %s loaded", __name__)
 
@@ -32,7 +32,7 @@ def initConfig(controllerObject):
                    "PROMPT"          : "Enter the IP address of the Horizon server",
                    "OPTION_LIST"     : [],
                    "VALIDATORS"      : [validators.validate_ssh],
-                   "DEFAULT_VALUE"   : utils.getLocalhostIP(),
+                   "DEFAULT_VALUE"   : utils.get_localhost_ip(),
                    "MASK_INPUT"      : False,
                    "LOOSE_VALIDATION": True,
                    "CONF_NAME"       : "CONFIG_HORIZON_HOST",

packstack/plugins/glance_200.py

@@ -7,7 +7,7 @@ import logging
 
 from packstack.installer import validators
 from packstack.installer import basedefs
-import packstack.installer.common_utils as utils
+from packstack.installer import utils
 
 from packstack.modules.ospluginutils import getManifestTemplate, appendManifestFile
 
@@ -16,7 +16,7 @@ controller = None
 
 # Plugin name
 PLUGIN_NAME = "OS-Glance"
-PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE)
+PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue')
 
 logging.debug("plugin %s loaded", __name__)
 
@@ -30,7 +30,7 @@ def initConfig(controllerObject):
                    "PROMPT"          : "Enter the IP address of the Glance server",
                    "OPTION_LIST"     : [],
                    "VALIDATORS"      : [validators.validate_ssh],
-                   "DEFAULT_VALUE"   : utils.getLocalhostIP(),
+                   "DEFAULT_VALUE"   : utils.get_localhost_ip(),
                    "MASK_INPUT"      : False,
                    "LOOSE_VALIDATION": True,
                    "CONF_NAME"       : "CONFIG_GLANCE_HOST",

packstack/plugins/keystone_100.py

@@ -8,7 +8,7 @@ import uuid
 
 from packstack.installer import validators
 from packstack.installer import basedefs
-import packstack.installer.common_utils as utils
+from packstack.installer import utils
 
 from packstack.modules.ospluginutils import getManifestTemplate, appendManifestFile
 
@@ -17,7 +17,7 @@ controller = None
 
 # Plugin name
 PLUGIN_NAME = "OS-Keystone"
-PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE)
+PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue')
 
 logging.debug("plugin %s loaded", __name__)
 
@@ -31,7 +31,7 @@ def initConfig(controllerObject):
                    "PROMPT"          : "Enter the IP address of the Keystone server",
                    "OPTION_LIST"     : [],
                    "VALIDATORS"      : [validators.validate_ssh],
-                   "DEFAULT_VALUE"   : utils.getLocalhostIP(),
+                   "DEFAULT_VALUE"   : utils.get_localhost_ip(),
                    "MASK_INPUT"      : False,
                    "LOOSE_VALIDATION": True,
                    "CONF_NAME"       : "CONFIG_KEYSTONE_HOST",
@@ -78,7 +78,7 @@ def initConfig(controllerObject):
 
     groupDict = { "GROUP_NAME"            : "KEYSTONE",
                   "DESCRIPTION"           : "Keystone Config parameters",
-                  "PRE_CONDITION"         : utils.returnYes,
+                  "PRE_CONDITION"         : lambda x: 'yes',
                   "PRE_CONDITION_MATCH"   : "yes",
                   "POST_CONDITION"        : False,
                   "POST_CONDITION_MATCH"  : True}

packstack/plugins/mysql_001.py

@@ -7,7 +7,7 @@ import logging
 
 from packstack.installer import validators
 from packstack.installer import basedefs
-import packstack.installer.common_utils as utils
+from packstack.installer import utils
 
 from packstack.modules.ospluginutils import getManifestTemplate, appendManifestFile
 
@@ -16,7 +16,7 @@ controller = None
 
 # Plugin name
 PLUGIN_NAME = "OS-MySQL"
-PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE)
+PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue')
 
 logging.debug("plugin %s loaded", __name__)
 
@@ -30,7 +30,7 @@ def initConfig(controllerObject):
                    "PROMPT"          : "Enter the IP address of the MySQL server",
                    "OPTION_LIST"     : [],
                    "VALIDATORS"      : [validators.validate_ssh],
-                   "DEFAULT_VALUE"   : utils.getLocalhostIP(),
+                   "DEFAULT_VALUE"   : utils.get_localhost_ip(),
                    "MASK_INPUT"      : False,
                    "LOOSE_VALIDATION": True,
                    "CONF_NAME"       : "CONFIG_MYSQL_HOST",
@@ -65,7 +65,7 @@ def initConfig(controllerObject):
 
     groupDict = { "GROUP_NAME"            : "MYSQL",
                   "DESCRIPTION"           : "MySQL Config parameters",
-                  "PRE_CONDITION"         : utils.returnYes,
+                  "PRE_CONDITION"         : lambda x: 'yes',
                   "PRE_CONDITION_MATCH"   : "yes",
                   "POST_CONDITION"        : False,
                   "POST_CONDITION_MATCH"  : True}

packstack/plugins/nagios_910.py

@@ -7,7 +7,7 @@ import logging
 
 from packstack.installer import validators
 from packstack.installer import basedefs, output_messages
-import packstack.installer.common_utils as utils
+from packstack.installer import utils
 
 from packstack.modules.ospluginutils import gethostlist,\
                                             getManifestTemplate,\
@@ -18,7 +18,7 @@ controller = None
 
 # Plugin name
 PLUGIN_NAME = "OS-Nagios"
-PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE)
+PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue')
 
 logging.debug("plugin %s loaded", __name__)
 
@@ -32,7 +32,7 @@ def initConfig(controllerObject):
                    "PROMPT"          : "Enter the IP address of the Nagios server",
                    "OPTION_LIST"     : [],
                    "VALIDATORS"      : [validators.validate_ssh],
-                   "DEFAULT_VALUE"   : utils.getLocalhostIP(),
+                   "DEFAULT_VALUE"   : utils.get_localhost_ip(),
                    "MASK_INPUT"      : False,
                    "LOOSE_VALIDATION": True,
                    "CONF_NAME"       : "CONFIG_NAGIOS_HOST",

packstack/plugins/nova_300.py

@@ -7,7 +7,7 @@ import uuid
 import logging
 
 from packstack.installer import validators
-import packstack.installer.common_utils as utils
+from packstack.installer import utils
 from packstack.installer.exceptions import ScriptRuntimeError
 
 from packstack.modules.ospluginutils import NovaConfig, getManifestTemplate, appendManifestFile, manifestfiles
@@ -29,7 +29,7 @@ def initConfig(controllerObject):
                    "PROMPT"          : "Enter the IP address of the Nova API service",
                    "OPTION_LIST"     : [],
                    "VALIDATORS"      : [validators.validate_ip, validators.validate_ssh],
-                   "DEFAULT_VALUE"   : utils.getLocalhostIP(),
+                   "DEFAULT_VALUE"   : utils.get_localhost_ip(),
                    "MASK_INPUT"      : False,
                    "LOOSE_VALIDATION": True,
                    "CONF_NAME"       : "CONFIG_NOVA_API_HOST",
@@ -41,7 +41,7 @@ def initConfig(controllerObject):
                    "PROMPT"          : "Enter the IP address of the Nova Cert service",
                    "OPTION_LIST"     : [],
                    "VALIDATORS"      : [validators.validate_ssh],
-                   "DEFAULT_VALUE"   : utils.getLocalhostIP(),
+                   "DEFAULT_VALUE"   : utils.get_localhost_ip(),
                    "MASK_INPUT"      : False,
                    "LOOSE_VALIDATION": True,
                    "CONF_NAME"       : "CONFIG_NOVA_CERT_HOST",
@@ -53,7 +53,7 @@ def initConfig(controllerObject):
                    "PROMPT"          : "Enter the IP address of the Nova VNC proxy",
                    "OPTION_LIST"     : [],
                    "VALIDATORS"      : [validators.validate_ssh],
-                   "DEFAULT_VALUE"   : utils.getLocalhostIP(),
+                   "DEFAULT_VALUE"   : utils.get_localhost_ip(),
                    "MASK_INPUT"      : False,
                    "LOOSE_VALIDATION": True,
                    "CONF_NAME"       : "CONFIG_NOVA_VNCPROXY_HOST",
@@ -65,7 +65,7 @@ def initConfig(controllerObject):
                    "PROMPT"          : "Enter a comma separated list of IP addresses on which to install the Nova Compute services",
                    "OPTION_LIST"     : [],
                    "VALIDATORS"      : [validators.validate_multi_ssh],
-                   "DEFAULT_VALUE"   : utils.getLocalhostIP(),
+                   "DEFAULT_VALUE"   : utils.get_localhost_ip(),
                    "MASK_INPUT"      : False,
                    "LOOSE_VALIDATION": True,
                    "CONF_NAME"       : "CONFIG_NOVA_COMPUTE_HOSTS",
@@ -89,7 +89,7 @@ def initConfig(controllerObject):
                    "PROMPT"          : "Enter the IP address of the Nova Network service",
                    "OPTION_LIST"     : [],
                    "VALIDATORS"      : [validators.validate_ip, validators.validate_ssh],
-                   "DEFAULT_VALUE"   : utils.getLocalhostIP(),
+                   "DEFAULT_VALUE"   : utils.get_localhost_ip(),
                    "MASK_INPUT"      : False,
                    "LOOSE_VALIDATION": True,
                    "CONF_NAME"       : "CONFIG_NOVA_NETWORK_HOST",
@@ -101,7 +101,7 @@ def initConfig(controllerObject):
                    "PROMPT"          : "Enter the IP address of the Nova Conductor service",
                    "OPTION_LIST"     : [],
                    "VALIDATORS"      : [validators.validate_ip, validators.validate_ssh],
-                   "DEFAULT_VALUE"   : utils.getLocalhostIP(),
+                   "DEFAULT_VALUE"   : utils.get_localhost_ip(),
                    "MASK_INPUT"      : False,
                    "LOOSE_VALIDATION": True,
                    "CONF_NAME"       : "CONFIG_NOVA_CONDUCTOR_HOST",
@@ -209,7 +209,7 @@ def initConfig(controllerObject):
                    "PROMPT"          : "Enter the IP address of the Nova Scheduler service",
                    "OPTION_LIST"     : [],
                    "VALIDATORS"      : [validators.validate_ssh],
-                   "DEFAULT_VALUE"   : utils.getLocalhostIP(),
+                   "DEFAULT_VALUE"   : utils.get_localhost_ip(),
                    "MASK_INPUT"      : False,
                    "LOOSE_VALIDATION": True,
                    "CONF_NAME"       : "CONFIG_NOVA_SCHED_HOST",

packstack/plugins/openstack_client_400.py

@@ -6,7 +6,7 @@ import logging
 
 from packstack.installer import validators
 from packstack.installer import basedefs, output_messages
-import packstack.installer.common_utils as utils
+from packstack.installer import utils
 
 from packstack.modules.ospluginutils import getManifestTemplate, appendManifestFile
 
@@ -15,7 +15,7 @@ controller = None
 
 # Plugin name
 PLUGIN_NAME = "OS-CLIENT"
-PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE)
+PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue')
 
 logging.debug("plugin %s loaded", __name__)
 
@@ -29,7 +29,7 @@ def initConfig(controllerObject):
                    "PROMPT"          : "Enter the IP address of the client server",
                    "OPTION_LIST"     : [],
                    "VALIDATORS"      : [validators.validate_ssh],
-                   "DEFAULT_VALUE"   : utils.getLocalhostIP(),
+                   "DEFAULT_VALUE"   : utils.get_localhost_ip(),
                    "MASK_INPUT"      : False,
                    "LOOSE_VALIDATION": True,
                    "CONF_NAME"       : "CONFIG_OSCLIENT_HOST",

packstack/plugins/postscript_949.py

@@ -6,7 +6,7 @@ import logging
 
 from packstack.installer import validators
 from packstack.installer import basedefs, output_messages
-import packstack.installer.common_utils as utils
+from packstack.installer import utils
 
 from packstack.modules.ospluginutils import gethostlist,\
                                             getManifestTemplate, \
@@ -28,7 +28,7 @@ def initConfig(controllerObject):
 
     groupDict = {"GROUP_NAME"            : "POSTSCRIPT",
                  "DESCRIPTION"           : "POSTSCRIPT Config parameters",
-                 "PRE_CONDITION"         : utils.returnYes,
+                 "PRE_CONDITION"         : lambda x: 'yes',
                  "PRE_CONDITION_MATCH"   : "yes",
                  "POST_CONDITION"        : False,
                  "POST_CONDITION_MATCH"  : True}

packstack/plugins/prescript_000.py

@@ -6,7 +6,7 @@ import uuid
 import logging
 
 from packstack.installer import validators
-import packstack.installer.common_utils as utils
+from packstack.installer import utils
 
 from packstack.modules.ospluginutils import gethostlist,\
                                             getManifestTemplate, \
@@ -123,7 +123,7 @@ def initConfig(controllerObject):
                  ]
     groupDict = { "GROUP_NAME"            : "GLOBAL",
                   "DESCRIPTION"           : "Global Options",
-                  "PRE_CONDITION"         : utils.returnYes,
+                  "PRE_CONDITION"         : lambda x: 'yes',
                   "PRE_CONDITION_MATCH"   : "yes",
                   "POST_CONDITION"        : False,
                   "POST_CONDITION_MATCH"  : True}

packstack/plugins/puppet_950.py

@@ -7,7 +7,7 @@ import os
 import platform
 import time
 
-import packstack.installer.common_utils as utils
+from packstack.installer import utils
 from packstack.installer import basedefs, output_messages
 from packstack.installer.exceptions import ScriptRuntimeError
 
@@ -20,7 +20,7 @@ controller = None
 
 # Plugin name
 PLUGIN_NAME = "OSPUPPET"
-PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE)
+PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue')
 
 logging.debug("plugin %s loaded", __name__)
 
@@ -37,7 +37,7 @@ def initConfig(controllerObject):
 
     groupDict = {"GROUP_NAME"            : "PUPPET",
                  "DESCRIPTION"           : "Puppet Config parameters",
-                 "PRE_CONDITION"         : utils.returnYes,
+                 "PRE_CONDITION"         : lambda x: 'yes',
                  "PRE_CONDITION_MATCH"   : "yes",
                  "POST_CONDITION"        : False,
                  "POST_CONDITION_MATCH"  : True}
@@ -162,7 +162,7 @@ def waitforpuppet(currently_running):
             # check the log file for errors
             validate_puppet_logfile(log)
             sys.stdout.write(("\r%s : " % log_file).ljust(basedefs.SPACE_LEN))
-            print ("[ " + utils.getColoredText(output_messages.INFO_DONE, basedefs.GREEN) + " ]")
+            print ("[ " + utils.color_text(output_messages.INFO_DONE, 'green') + " ]")
 
 
 def applyPuppetManifest():

packstack/plugins/qpid_002.py

@@ -6,7 +6,7 @@ import logging
 
 from packstack.installer import validators
 from packstack.installer import basedefs
-import packstack.installer.common_utils as utils
+from packstack.installer import utils
 
 from packstack.modules.ospluginutils import getManifestTemplate, appendManifestFile
 
@@ -15,7 +15,7 @@ controller = None
 
 # Plugin name
 PLUGIN_NAME = "OS-QPID"
-PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE)
+PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue')
 
 logging.debug("plugin %s loaded", __name__)
 
@@ -29,7 +29,7 @@ def initConfig(controllerObject):
                    "PROMPT"          : "Enter the IP address of the QPID service",
                    "OPTION_LIST"     : [],
                    "VALIDATORS"      : [validators.validate_ssh],
-                   "DEFAULT_VALUE"   : utils.getLocalhostIP(),
+                   "DEFAULT_VALUE"   : utils.get_localhost_ip(),
                    "MASK_INPUT"      : False,
                    "LOOSE_VALIDATION": True,
                    "CONF_NAME"       : "CONFIG_QPID_HOST",

packstack/plugins/serverprep_901.py

@@ -9,7 +9,7 @@ import datetime
 import platform
 
 from packstack.installer import basedefs
-from packstack.installer import common_utils as utils
+from packstack.installer import utils
 from packstack.installer import validators
 from packstack.installer.exceptions import InstallError
 
@@ -21,7 +21,7 @@ controller = None
 
 # Plugin name
 PLUGIN_NAME = "OS-SERVERPREPARE"
-PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE)
+PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue')
 
 logging.debug("plugin %s loaded", __name__)
 
@@ -241,7 +241,7 @@ def initConfig(controllerObject):
     conf_groups = [
              {"GROUP_NAME"            : "SERVERPREPARE",
               "DESCRIPTION"           : "Server Prepare Configs ",
-              "PRE_CONDITION"         : utils.returnYes,
+              "PRE_CONDITION"         : lambda x: 'yes',
               "PRE_CONDITION_MATCH"   : "yes",
               "POST_CONDITION"        : False,
               "POST_CONDITION_MATCH"  : True},

packstack/plugins/sshkeys_000.py

@@ -10,7 +10,7 @@ import tempfile
 from packstack.installer import processors
 from packstack.installer import validators
 from packstack.installer import basedefs
-import packstack.installer.common_utils as utils
+from packstack.installer import utils
 
 from packstack.modules.ospluginutils import gethostlist
 
@@ -19,7 +19,7 @@ controller = None
 
 # Plugin name
 PLUGIN_NAME = "OS-SSHKEYS"
-PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE)
+PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue')
 
 logging.debug("plugin %s loaded", __name__)
 
@@ -45,7 +45,7 @@ def initConfig(controllerObject):
 
     groupDict = { "GROUP_NAME"            : "SSHKEY",
                   "DESCRIPTION"           : "SSH Configs ",
-                  "PRE_CONDITION"         : utils.returnYes,
+                  "PRE_CONDITION"         : lambda x: 'yes',
                   "PRE_CONDITION_MATCH"   : "yes",
                   "POST_CONDITION"        : False,
                   "POST_CONDITION_MATCH"  : True}

packstack/plugins/swift_600.py

@@ -8,7 +8,7 @@ import os
 
 from packstack.installer import validators
 from packstack.installer import basedefs
-import packstack.installer.common_utils as utils
+from packstack.installer import utils
 
 from packstack.modules.ospluginutils import getManifestTemplate, appendManifestFile, manifestfiles
 
@@ -17,7 +17,7 @@ controller = None
 
 # Plugin name
 PLUGIN_NAME = "OS-SWIFT"
-PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE)
+PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue')
 
 logging.debug("plugin %s loaded", __name__)
 
@@ -31,7 +31,7 @@ def initConfig(controllerObject):
                    "PROMPT"          : "Enter the IP address of the Swift proxy service",
                    "OPTION_LIST"     : [],
                    "VALIDATORS"      : [validators.validate_ip, validators.validate_ssh],
-                   "DEFAULT_VALUE"   : utils.getLocalhostIP(),
+                   "DEFAULT_VALUE"   : utils.get_localhost_ip(),
                    "MASK_INPUT"      : False,
                    "LOOSE_VALIDATION": True,
                    "CONF_NAME"       : "CONFIG_SWIFT_PROXY_HOSTS", #XXX: Shouldn't be here CONFIG_SWIFT_PROXY_HOST?
@@ -55,7 +55,7 @@ def initConfig(controllerObject):
                    "PROMPT"          : "Enter the Swift Storage servers e.g. host/dev,host/dev",
                    "OPTION_LIST"     : [],
                    "VALIDATORS"      : [validators.validate_not_empty, validate_storage],
-                   "DEFAULT_VALUE"   : utils.getLocalhostIP(),
+                   "DEFAULT_VALUE"   : utils.get_localhost_ip(),
                    "MASK_INPUT"      : False,
                    "LOOSE_VALIDATION": True,
                    "CONF_NAME"       : "CONFIG_SWIFT_STORAGE_HOSTS",

tests/installer/test_utils.py

@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+
+"""
+Test cases for packstack.installer.utils module.
+"""
+
+import shutil
+import tempfile
+from unittest import TestCase
+
+from ..test_base import PackstackTestCaseMixin
+from packstack.installer.utils import *
+from packstack.installer.utils.strings import STR_MASK
+from packstack.installer.exceptions import ExecuteRuntimeError
+
+
+cnt = 0
+
+
+class ParameterTestCase(PackstackTestCaseMixin, TestCase):
+    def setUp(self):
+        # Creating a temp directory that can be used by tests
+        self.tempdir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        # remove the temp directory
+        shutil.rmtree(self.tempdir)
+
+    def test_sorteddict(self):
+        """Test packstack.installer.utils.datastructures.SortedDict"""
+        sdict = SortedDict()
+        sdict['1'] = 1
+        sdict['2'] = 2
+        sdict.update(SortedDict([('3', 3), ('4', 4), ('5', 5)]))
+        self.assertListEqual(sdict.keys(), ['1', '2', '3', '4', '5'])
+        self.assertListEqual(sdict.values(), [1, 2, 3, 4, 5])
+
+    def test_retry(self):
+        """Test packstack.installer.utils.decorators.retry"""
+
+        @retry(count=3, delay=0, retry_on=ValueError)
+        def test_sum():
+            global cnt
+            cnt += 1
+            raise ValueError
+
+        global cnt
+        cnt = 0
+
+        try:
+            test_sum()
+        except ValueError:
+            pass
+        self.assertEqual(cnt, 4)
+        self.assertRaises(ValueError, test_sum)
+
+    def test_network(self):
+        """Test packstack.installer.utils.network functions"""
+        self.assertIn(host2ip('google-public-dns-a.google.com'),
+                      ['8.8.8.8', '2001:4860:4860::8888'])
+        self.assertIn(host2ip('localhost', allow_localhost=True),
+                      ['127.0.0.1', '::1'])
+
+    def test_shell(self):
+        """Test packstack.installer.utils.shell functions"""
+        rc, out = execute(['echo', 'this is test'])
+        self.assertEqual(out.strip(), 'this is test')
+        rc, out = execute('echo "this is test"', use_shell=True)
+        self.assertEqual(out.strip(), 'this is test')
+        try:
+            execute('echo "mask the password" && exit 1',
+                    use_shell=True, mask_list=['password'])
+            raise AssertionError('Masked execution failed.')
+        except ExecuteRuntimeError, ex:
+            should_be = ('Failed to execute command: '
+                         'echo "mask the %s" && exit 1' % STR_MASK)
+            self.assertEqual(str(ex), should_be)
+
+        script = ScriptRunner()
+        script.append('echo "this is test"')
+        rc, out = script.execute()
+        self.assertEqual(out.strip(), 'this is test')
+
+    def test_strings(self):
+        """Test packstack.installer.utils.strings functions"""
+        self.assertEqual(color_text('test text', 'red'),
+                        '\033[0;31mtest text\033[0m')
+        self.assertEqual(mask_string('test text', mask_list=['text']),
+                         'test %s' % STR_MASK)
+        masked = mask_string("test '\\''text'\\''",
+                             mask_list=["'text'"],
+                             replace_list=[("'", "'\\''")])
+        self.assertEqual(masked, 'test %s' % STR_MASK)