Program running cleanup and some additional classes added to make python running easier for each component.

This commit is contained in:
Joshua Harlow 2012-01-17 15:24:57 -08:00
parent 4e1f124d2f
commit cb5fefd06c
8 changed files with 165 additions and 87 deletions

@ -28,7 +28,9 @@ from Trace import (TraceWriter, TraceReader,
touch_trace, parse_fn,
IN_TRACE, PY_TRACE, START_TRACE)
import Runner
from runners import Foreground, Screen
from runners.Foreground import (ForegroundRunner)
from runners.Screen import ScreenRunner
from Exceptions import (StopException, StartException, InstallException)
LOG = Logger.getLogger("install.component")
@ -249,40 +251,51 @@ class PythonUninstallComponent(PkgUninstallComponent):
class ProgramRuntime(ComponentBase, RuntimeComponent):
#this here determines how we start and stop and
#what classes handle different running/stopping types
STARTER_CLS_MAPPING = {
Foreground.RUN_TYPE: ForegroundRunner,
Screen.RUN_TYPE: ScreenRunner,
}
STOPPER_CLS_MAPPING = {
Foreground.RUN_TYPE: ForegroundRunner,
Screen.RUN_TYPE: ScreenRunner,
}
def __init__(self, component_name, *args, **kargs):
ComponentBase.__init__(self, component_name, *args, **kargs)
self.foreground = kargs.get("foreground", True)
self.run_type = kargs.get("run_type", Foreground.RUN_TYPE)
self.tracereader = TraceReader(self.tracedir, IN_TRACE)
self.tracewriter = TraceWriter(self.tracedir, START_TRACE)
self.starttracereader = TraceReader(self.tracedir, START_TRACE)
self.check_installed_pkgs = kargs.get("check_installed_pkgs", True)
def _getstartercls(self):
if(self.foreground):
return ForegroundRunner
else:
raise NotImplementedError("Can not yet start in non-foreground mode")
def _getstartercls(self, start_mode):
if(start_mode not in ProgramRuntime.STARTER_CLS_MAPPING):
raise NotImplementedError("Can not yet start %s mode" % (start_mode))
return ProgramRuntime.STARTER_CLS_MAPPING.get(start_mode)
def _getstoppercls(self, starttype):
if(starttype == Foreground.RUN_TYPE):
return ForegroundRunner
else:
raise NotImplementedError("Can not yet stop type [%s]" % (starttype))
def _getstoppercls(self, stop_mode):
if(stop_mode not in ProgramRuntime.STOPPER_CLS_MAPPING):
raise NotImplementedError("Can not yet stop %s mode" % (stop_mode))
return ProgramRuntime.STOPPER_CLS_MAPPING.get(stop_mode)
def _was_installed(self):
pkgsinstalled = self.tracereader.packages_installed()
if(len(pkgsinstalled) == 0):
return False
if(not self.check_installed_pkgs):
return True
if(len(self.tracereader.packages_installed())):
return True
return False
def _get_apps_to_start(self):
raise NotImplementedError()
def _get_app_options(self, app, params):
return list()
def _get_app_options(self, app):
return None
def _get_param_map(self, app=None):
replacements = dict()
replacements['ROOT'] = self.appdir
return replacements
return {
'ROOT': self.appdir,
}
def start(self):
#ensure it was installed
@ -299,7 +312,12 @@ class ProgramRuntime(ComponentBase, RuntimeComponent):
for app in apps:
#adjust the program options now that we have real locations
params = self._get_param_map(app)
program_opts = self._get_app_options(app, params)
program_opts = self._get_app_options(app)
if(params and program_opts):
adjusted_opts = list()
for opt in program_opts:
adjusted_opts.append(param_replace(opt, params))
program_opts = adjusted_opts
LOG.info("Starting %s with options [%s]" % (app, ", ".join(program_opts)))
#start it with the given settings
fn = starter.start(app, app, *program_opts, app_dir=self.appdir, trace_dir=self.tracedir)
@ -337,7 +355,7 @@ class ProgramRuntime(ComponentBase, RuntimeComponent):
#did we find a class that can do it?
if(killcls):
#we can try to stop it
LOG.info("Stopping %s with %s in %s" % (name, runtype, self.tracedir))
LOG.info("Stopping %s" % (name))
#create an instance of the killer class and attempt to stop
killer = killcls()
killer.stop(name, trace_dir=self.tracedir)
@ -347,3 +365,18 @@ class ProgramRuntime(ComponentBase, RuntimeComponent):
fn = self.starttracereader.trace_fn
LOG.info("Deleting trace file %s" % (fn))
unlink(fn)
class PythonRuntime(ProgramRuntime):
def __init__(self, component_name, *args, **kargs):
ProgramRuntime.__init__(self, component_name, *args, **kargs)
def _was_installed(self):
parent_result = ProgramRuntime._was_installed(self)
if(not parent_result):
return False
python_installed = self.tracereader.py_listing()
if(len(python_installed) == 0):
return False
else:
return True

@ -27,11 +27,13 @@ PW_TMPL = "Enter a password for %s: "
ENV_PAT = re.compile(r"^\s*\$\{([\w\d]+):\-(.*)\}\s*$")
class EnvConfigParser(ConfigParser.RawConfigParser):
def __init__(self):
ConfigParser.RawConfigParser.__init__(self)
self.pws = dict()
self.configs_fetched = dict()
self.dbdsns = dict()
def _makekey(self, section, option):
return option + "@" + section
@ -48,6 +50,24 @@ class EnvConfigParser(ConfigParser.RawConfigParser):
self.configs_fetched[key] = v
return v
def __str__(self):
str_repr = ""
if(len(self.pws)):
str_repr += "Passwords:" + os.linesep
for (k,v) in self.pws.items():
str_repr += "\t" + str(k) + " = " + str(v) + os.linesep
if(len(self.configs_fetched)):
str_repr += "Configs:" + os.linesep
for (k,v) in self.configs_fetched.items():
if(k in self.pws):
continue
str_repr += "\t" + str(k) + " = " + str(v) + os.linesep
if(len(self.dbdsns)):
str_repr += "Data source names:" + os.linesep
for (k, v) in self.dbdsns.items():
str_repr += "\t" + str(k) + " = " + str(v) + os.linesep
return str_repr
def _get_special(self, section, option):
key = self._makekey(section, option)
v = ConfigParser.RawConfigParser.get(self, section, option)
@ -69,6 +89,41 @@ class EnvConfigParser(ConfigParser.RawConfigParser):
else:
return v
def get_dbdsn(dbname):
user = self.get("db", "sql_user")
host = self.get("db", "sql_host")
port = self.get("db", "port")
pw = self.getpw("passwords", "sql")
#check the dsn cache
if(dbname in self.dbdsns):
return self.dbdsns[dbname]
#form the dsn (from components we have...)
#dsn = "<driver>://<username>:<password>@<host>:<port>/<database>"
if(not host):
msg = "Unable to fetch a database dsn - no host found"
raise BadParamException(msg)
driver = self.get("db", "type")
if(not driver):
msg = "Unable to fetch a database dsn - no driver type found"
raise BadParamException(msg)
dsn = driver + "://"
if(user):
dsn += user
if(password):
dsn += ":" + password
if(user or password):
dsn += "@"
dsn += host
if(port):
dsn += ":" + port
if(dbname):
dsn += "/" + dbname
else:
dsn += "/"
#store for later...
self.dbdsns[dbname] = dsn
return dsn
def getpw(self, section, option):
key = self._makekey(section, option)
pw = self.pws.get(key)

@ -19,11 +19,9 @@ import os.path
import Logger
from Component import (PythonUninstallComponent,
PythonInstallComponent,
ProgramRuntime)
PythonRuntime)
from Util import (GLANCE,
get_host_ip,
param_replace, get_dbdsn,
)
get_host_ip, param_replace)
from Shell import (deldir, mkdirslist, unlink,
joinpths, touch_file)
@ -37,7 +35,6 @@ CONFIGS = [API_CONF, REG_CONF]
DB_NAME = "glance"
#what to start
APPS_TO_START = ['glance-api', 'glance-registry']
APP_OPTIONS = {
'glance-api': ['--config-file', joinpths('%ROOT%', "etc", API_CONF)],
'glance-registry': ['--config-file', joinpths('%ROOT%', "etc", REG_CONF)]
@ -51,29 +48,16 @@ class GlanceUninstaller(PythonUninstallComponent):
self.cfgdir = joinpths(self.appdir, CONFIG_ACTUAL_DIR)
class GlanceRuntime(ProgramRuntime):
class GlanceRuntime(PythonRuntime):
def __init__(self, *args, **kargs):
ProgramRuntime.__init__(self, TYPE, *args, **kargs)
PythonRuntime.__init__(self, TYPE, *args, **kargs)
self.cfgdir = joinpths(self.appdir, CONFIG_ACTUAL_DIR)
def _was_installed(self):
pres = ProgramRuntime._was_installed(self)
if(pres == False):
return False
pylisting = self.tracereader.py_listing()
if(pylisting and len(pylisting)):
return True
return False
def _get_apps_to_start(self):
raise list(APPS_TO_START)
return sorted(APP_OPTIONS.keys())
def _get_app_options(self, app, params):
opts = list()
if(app in APP_OPTIONS):
for opt in APP_OPTIONS.get(app):
opts.append(param_replace(opt, params))
return opts
def _get_app_options(self, app):
return APP_OPTIONS.get(app)
class GlanceInstaller(PythonInstallComponent):
@ -84,11 +68,11 @@ class GlanceInstaller(PythonInstallComponent):
self.cfgdir = joinpths(self.appdir, CONFIG_ACTUAL_DIR)
def _get_download_location(self):
uri = self.gitloc
branch = self.brch
return (uri, branch)
#where we get glance from
return (self.gitloc, self.brch)
def _get_config_files(self):
#these are the config files we will be adjusting
return list(CONFIGS)
def _config_adjust(self, contents, fn):
@ -144,7 +128,7 @@ class GlanceInstaller(PythonInstallComponent):
mp['DEST'] = self.appdir
mp['SYSLOG'] = self.cfg.getboolean("default", "syslog")
mp['SERVICE_TOKEN'] = self.cfg.getpw("passwords", "service_token")
mp['SQL_CONN'] = get_dbdsn(self.cfg, DB_NAME)
mp['SQL_CONN'] = self.cfg.get_dbdsn(DB_NAME)
hostip = get_host_ip(self.cfg)
mp['SERVICE_HOST'] = hostip
mp['HOST_IP'] = hostip

@ -20,12 +20,13 @@ import Pip
from Util import (KEYSTONE,
CONFIG_DIR,
NOVA, GLANCE, SWIFT,
get_dbdsn,
get_host_ip,
execute_template)
execute_template,
param_replace)
import Logger
import Db
from Component import (PythonUninstallComponent, PythonInstallComponent, ProgramRuntime)
from Component import (PythonUninstallComponent,
PythonInstallComponent, PythonRuntime)
from Shell import (mkdirslist, unlink, touch_file, joinpths)
LOG = Logger.getLogger("install.keystone")
@ -35,6 +36,11 @@ CONFIGS = [ROOT_CONF]
BIN_DIR = "bin"
DB_NAME = "keystone"
#what to start
APP_OPTIONS = {
'keystone': ['--config-file', joinpths('%ROOT%', "config", ROOT_CONF), "--verbose"],
}
class KeystoneUninstaller(PythonUninstallComponent):
def __init__(self, *args, **kargs):
@ -110,7 +116,7 @@ class KeystoneInstaller(PythonInstallComponent):
#params with actual values
mp = dict()
mp['DEST'] = self.appdir
mp['SQL_CONN'] = get_dbdsn(self.cfg, DB_NAME)
mp['SQL_CONN'] = self.cfg.get_dbdsn(DB_NAME)
mp['ADMIN_PASSWORD'] = self.cfg.getpw('passwords', 'horizon_keystone_admin')
mp['HOST_IP'] = get_host_ip(self.cfg)
mp['SERVICE_TOKEN'] = self.cfg.getpw("passwords", "service_token")
@ -119,12 +125,18 @@ class KeystoneInstaller(PythonInstallComponent):
return mp
class KeystoneRuntime(ProgramRuntime):
class KeystoneRuntime(PythonRuntime):
def __init__(self, *args, **kargs):
ProgramRuntime.__init__(self, TYPE, *args, **kargs)
PythonRuntime.__init__(self, TYPE, *args, **kargs)
self.cfgdir = joinpths(self.appdir, CONFIG_DIR)
self.bindir = joinpths(self.appdir, BIN_DIR)
def _get_apps_to_start(self):
return sorted(APP_OPTIONS.keys())
def _get_app_options(self, app):
return APP_OPTIONS.get(app)
# Keystone setup commands are the the following
def _keystone_setup_cmds(components):

@ -42,7 +42,6 @@ RHEL6 = "rhel-6"
MASTER_BRANCH = "master"
#other constants
DB_DSN = '%s://%s:%s@%s/%s'
PRE_INSTALL = 'pre-install'
POST_INSTALL = 'post-install'
IPV4 = 'IPv4'
@ -311,14 +310,6 @@ def create_regex(format):
return re.compile(toberegex, flags)
def get_dbdsn(cfg, dbname):
user = cfg.get("db", "sql_user")
host = cfg.get("db", "sql_host")
dbtype = cfg.get("db", "type")
pw = cfg.getpw("passwords", "sql")
return DB_DSN % (dbtype, user, pw, host, dbname)
def determine_os():
os = None
plt = platform.platform()

@ -18,6 +18,7 @@ import sys
import resource
import signal
import errno
import time
import Runner
import Util
@ -32,6 +33,7 @@ import Trace
# Maximum for the number of available file descriptors (when not found)
MAXFD = 2048
MAX_KILL_TRY = 4
SLEEP_TIME = 1
LOG = Logger.getLogger("install.runners.foreground")
@ -67,7 +69,9 @@ class ForegroundRunner(Runner.Runner):
if(ec == errno.ESRCH):
killed = True
break
lastmsg = msg
else:
lastmsg = msg
time.sleep(SLEEP_TIME)
#trash the files
if(killed):
LOG.info("Killed pid %s in %s attempts" % (str(pid), str(attempts)))

@ -26,9 +26,9 @@ from Shell import (execute)
LOG = Logger.getLogger("install.screen")
SCREEN_MAKE = ['screen', '-d', '-m', '-S', '%NAME%', '-t', '%NAME%']
NAME_POSTFIX = ".devstack"
RUN_TYPE = "SCREEN"
class Screen(Runner.Runner):
class ScreenRunner(Runner.Runner):
def __init__(self):
Runner.Runner.__init__(self)

41
stack

@ -116,19 +116,8 @@ def get_config(action):
def print_cfgs(cfg, action):
pws = (cfg.pws)
if(pws and len(pws)):
LOG.info("After %s your fetched/generated passwords are:" % (action))
for (pwkey, val) in pws.items():
LOG.info("\t%s = %s" % (pwkey, val))
configs_fetched = (cfg.configs_fetched)
if(configs_fetched and len(configs_fetched)):
LOG.info("After %s your other fetched configuration was:" % (action))
for (key, val) in configs_fetched.items():
if(key in pws):
#already shown
continue
LOG.info("\t%s = %s" % (key, val))
LOG.info("After %s your config is:" % (action))
LOG.info(str(cfg).rstrip())
def runner(action_name, component_set, distro, root_dir, program_args):
@ -141,6 +130,7 @@ def runner(action_name, component_set, distro, root_dir, program_args):
LOG.info("Will %s [%s] using root directory %s" % (action_name, ", ".join(component_set), root_dir))
results = list()
class_lookup = ACTION_CLASSES.get(action_name)
force = program_args.get('force', False)
for c in component_set:
klass = class_lookup.get(c)
instance = klass(components=component_set, distro=distro, pkg=pkg_manager, cfg=config, root=root_dir)
@ -151,26 +141,35 @@ def runner(action_name, component_set, distro, root_dir, program_args):
instance.configure()
LOG.info("Installing %s." % (c))
trace = instance.install()
LOG.info("Finished install of %s - check %s for traces of what happened." % (c, trace))
results.append(trace)
if(trace):
LOG.info("Finished install of %s - check %s for traces of what happened." % (c, trace))
results.append(trace)
else:
LOG.info("Finished install of %s" % (c))
elif(action_name == STOP):
LOG.info("Stopping %s." % (c))
instance.stop()
LOG.info("Finished stop of %s" % (c))
try:
LOG.info("Stopping %s." % (c))
instance.stop()
LOG.info("Finished stop of %s" % (c))
except NoTraceException, e:
if(force):
LOG.info("Passing on stopping %s since no trace file was found." % (c))
else:
raise
elif(action_name == START):
LOG.info("Starting %s." % (c))
trace_locs = instance.start() or list()
LOG.info("Finished start of %s - check [%s] for traces of what happened." % (c, ", ".join(trace_locs)))
results = results + trace_locs
if(trace_locs):
results = results + trace_locs
elif(action_name == UNINSTALL):
ignore_traces_missing = program_args.get('force', False)
try:
LOG.info("Unconfiguring %s." % (c))
instance.unconfigure()
LOG.info("Uninstalling %s." % (c))
instance.uninstall()
except NoTraceException, e:
if(ignore_traces_missing):
if(force):
LOG.info("Passing on uninstalling %s since no trace file was found." % (c))
else:
raise