diff --git a/conf/personas/devstack.sh.yaml b/conf/personas/devstack.sh.yaml index 7ed3ba0a..cb28547b 100644 --- a/conf/personas/devstack.sh.yaml +++ b/conf/personas/devstack.sh.yaml @@ -1,10 +1,12 @@ --- -created_on: Wed, 14 Mar 2012 17:28:24 -0700 -description: Devstack.sh matching component installation (as of the above date). +description: Devstack.sh matching component installation. supports: - rhel-6 - ubuntu-oneiric - fedora-16 +options: + glance: + - load-images components: - db - rabbit-mq diff --git a/devstack/component.py b/devstack/component.py index 525daf99..6235f50f 100644 --- a/devstack/component.py +++ b/devstack/component.py @@ -93,10 +93,10 @@ class ComponentBase(object): knowns = self.known_subsystems() for s in self.desired_subsystems: if s not in knowns: - raise RuntimeError("Unknown subsystem %r requested" % (s)) + raise ValueError("Unknown subsystem %r requested" % (s)) for s in self.subsystem_info.keys(): if s not in knowns: - raise RuntimeError("Unknown subsystem %r provided" % (s)) + raise ValueError("Unknown subsystem %r provided" % (s)) def known_subsystems(self): return list() diff --git a/devstack/components/swift.py b/devstack/components/swift.py index 7db97235..3fe49179 100644 --- a/devstack/components/swift.py +++ b/devstack/components/swift.py @@ -62,7 +62,8 @@ CONFIG_DIR = 'etc' LOG_DIR = 'logs' # Config keys we warm up so u won't be prompted later -WARMUP_PWS = ['service_token', 'swift_hash'] +WARMUP_PWS = [('service_token', 'the service admin token'), + ('swift_hash', 'the random unique string for your swift cluster')] class SwiftUninstaller(comp.PythonUninstallComponent): @@ -107,8 +108,8 @@ class SwiftInstaller(comp.PythonInstallComponent): return list(CONFIGS) def warm_configs(self): - for pw_key in WARMUP_PWS: - self.pw_gen.get_password(pw_key) + for (pw_key, prompt) in WARMUP_PWS: + self.pw_gen.get_password(pw_key, prompt) def _get_param_map(self, config_fn): return { diff --git a/devstack/distro.py b/devstack/distro.py index 388edb27..df282b4a 100644 --- a/devstack/distro.py +++ b/devstack/distro.py @@ -40,14 +40,22 @@ class Distro(object): raise RuntimeError( 'Did not find any distro definition files in %s' % path) - for filename in input_files: + for fn in input_files: + cls_kvs = None + filename = sh.abspth(fn) + LOG.audit("Attempting to load distro definition from [%s]" % (filename)) try: with open(filename, 'r') as f: - data = yaml.load(f) - results.append(cls(**data)) + cls_kvs = yaml.load(f) except (IOError, yaml.YAMLError) as err: LOG.warning('Could not load distro definition from %s: %s', filename, err) + if cls_kvs is not None: + try: + results.append(cls(**cls_kvs)) + except Exception as err: + LOG.warning('Could not initialize instance %s using parameter map %s: %s', + cls, cls_kvs, err) return results @classmethod @@ -67,6 +75,7 @@ class Distro(object): 'No platform configuration data for %s (%s)' % (plt, distname)) + @logging.log_debug def __init__(self, name, distro_pattern, packager_name, commands, components): self.name = name self._distro_pattern = re.compile(distro_pattern, re.IGNORECASE) @@ -74,9 +83,6 @@ class Distro(object): self._commands = commands self._components = components - def __repr__(self): - return "\"%s\" using packager \"%s\"" % (self.name, self._packager_name) - def get_command(self, key, *more_keys, **kargs): """ Gets a end object for a given set of keys """ root = self._commands diff --git a/devstack/log.py b/devstack/log.py index de1966fb..807ea542 100644 --- a/devstack/log.py +++ b/devstack/log.py @@ -80,12 +80,10 @@ def getLogger(name='devstack'): def log_debug(f): @functools.wraps(f) - def wrapper(*args, **kw): - if root.isEnabledFor(debug): - logging.debug('%s(%s, %s) ->', f.func_name, str(args), str(kw)) - rv = f(*args, **kw) - if root.isEnabledFor(debug): - logging.debug(pprint.pformat(rv, indent=2)) - logging.debug('') + def wrapper(*args, **kargs): + logger = getLogger() + logger.debug('%s(%s, %s) ->', f.func_name, str(args), str(kargs)) + rv = f(*args, **kargs) + logger.debug("<- %s" % (pprint.pformat(rv, indent=2))) return rv return wrapper diff --git a/devstack/packager.py b/devstack/packager.py index 99f35500..c2822216 100644 --- a/devstack/packager.py +++ b/devstack/packager.py @@ -21,6 +21,8 @@ LOG = logging.getLogger("devstack.packager") class Packager(object): + + @logging.log_debug def __init__(self, distro, keep_packages): self.distro = distro self.keep_packages = keep_packages diff --git a/devstack/persona.py b/devstack/persona.py new file mode 100644 index 00000000..c4403789 --- /dev/null +++ b/devstack/persona.py @@ -0,0 +1,86 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import yaml + +from devstack import exceptions as excp +from devstack import log as logging +from devstack import shell as sh + +LOG = logging.getLogger("devstack.persona") + + +class Persona(object): + + @classmethod + def load_file(cls, fn): + persona_fn = sh.abspth(fn) + LOG.audit("Loading persona from file [%s]", persona_fn) + cls_kvs = None + try: + with open(persona_fn, "r") as fh: + cls_kvs = yaml.load(fh.read()) + except (IOError, yaml.YAMLError) as err: + LOG.warning('Could not load persona definition from %s: %s', + persona_fn, err) + instance = None + if cls_kvs is not None: + try: + cls_kvs['source'] = persona_fn + instance = cls(**cls_kvs) + except Exception as err: + LOG.warning('Could not initialize instance %s using parameter map %s: %s', + cls, cls_kvs, err) + return instance + + @logging.log_debug + def __init__(self, description, + supports, + components, + subsystems, + options, + **kargs): + self.distro_support = supports + self.wanted_components = components + self.source = kargs.get('source') # May not always be there (ie if from a stream...) + self.wanted_subsystems = subsystems + self.description = description + self.component_options = options + + def __str__(self): + info = "%s" % (self.description) + if self.source: + info += " from source %s:" % (self.source) + if self.wanted_subsystems: + info += " with desired subsystems (%s)" % (self.wanted_subsystems) + if self.wanted_components: + info += " with desired components (%s)" % (", ".join(self.wanted_components)) + if self.component_options: + info += " with desired component options (%s)" % (", ".join(self.component_options)) + if self.distro_support: + info += " which 'should' work on distros (%s)" % (", ".join(self.distro_support)) + return info + + def verify(self, distro): + # Some sanity checks against the given distro + d_name = distro.name + if d_name not in self.distro_support: + msg = "Distro %s not supported" % (d_name) + raise excp.ConfigException(msg) + for c in self.wanted_components: + if not distro.known_component(c): + raise RuntimeError("Distro %s does not support component %s" % + (d_name, c)) diff --git a/devstack/progs/actions.py b/devstack/progs/actions.py index 32d09cf3..a4bcd242 100644 --- a/devstack/progs/actions.py +++ b/devstack/progs/actions.py @@ -14,14 +14,11 @@ # License for the specific language governing permissions and limitations # under the License.. -import yaml - from devstack import env_rc from devstack import exceptions as excp from devstack import log as logging from devstack import settings from devstack import shell as sh -from devstack import passwords LOG = logging.getLogger("devstack.progs.actions") @@ -138,16 +135,16 @@ PREQ_ACTIONS = { class ActionRunner(object): - def __init__(self, distro, action, cfg, **kargs): + def __init__(self, distro, action, + cfg, pw_gen, pkg_manager, + **kargs): self.distro = distro self.action = action self.cfg = cfg - self.pw_gen = passwords.PasswordGenerator(self.cfg, kargs.get('prompt_for_passwords', True)) - pkg_cls = distro.get_packager_factory() - self.keep_old = kargs.get('keep_old') - self.pkg_manager = pkg_cls(self.distro, self.keep_old) + self.pw_gen = pw_gen + self.pkg_manager = pkg_manager + self.keep_old = kargs.get('keep_old', False) self.force = kargs.get('force', False) - self.kargs = kargs def _apply_reverse(self, action, component_order): adjusted_order = list(component_order) @@ -155,33 +152,9 @@ class ActionRunner(object): adjusted_order.reverse() return adjusted_order - def _load_persona(self, persona_fn): - persona_fn = sh.abspth(persona_fn) - LOG.audit("Loading persona from file [%s]", persona_fn) - contents = '' - with open(persona_fn, "r") as fh: - contents = fh.read() - return self._verify_persona(yaml.load(contents), persona_fn) - - def _verify_persona(self, persona, fn): - # Some sanity checks - try: - if self.distro.name not in persona['supports']: - raise RuntimeError("Persona does not support distro %s" - % (self.distro.name)) - for c in persona['components']: - if not self.distro.known_component(c): - raise RuntimeError("Distro %s does not support component %s" % - (self.distro.name, c)) - except (KeyError, RuntimeError) as e: - msg = ("Could not validate persona defined in [%s] due to: %s" - % (fn, e)) - raise excp.ConfigException(msg) - return persona - def _construct_instances(self, persona, action, root_dir): - components = persona['components'] # Required - desired_subsystems = persona.get('subsystems', dict()) # Not required + components = persona.wanted_components + desired_subsystems = persona.wanted_subsystems or dict() instances = dict() for c in components: (cls, my_info) = self.distro.extract_component(c, action) @@ -198,10 +171,7 @@ class ActionRunner(object): for (k, v) in my_info.items(): if k not in cls_kvs: cls_kvs[k] = v - LOG.debug("Using arg map %s", cls_kvs) - cls_args = list() - LOG.debug("Using arg list %s", cls_args) - instances[c] = cls(*cls_args, **cls_kvs) + instances[c] = cls(**cls_kvs) return instances def _verify_components(self, component_order, instances): @@ -243,7 +213,6 @@ class ActionRunner(object): raise def _run_action(self, persona, action, root_dir): - LOG.info("Running action [%s] using root directory [%s]" % (action, root_dir)) instances = self._construct_instances(persona, action, root_dir) if action in PREQ_ACTIONS: (check_functor, preq_action) = PREQ_ACTIONS[action] @@ -255,7 +224,7 @@ class ActionRunner(object): LOG.info("Activating prerequisite action [%s] requested by (%s) components." % (preq_action, ", ".join(checks_passed_components))) self._run_action(persona, preq_action, root_dir) - component_order = self._apply_reverse(action, persona['components']) + component_order = self._apply_reverse(action, persona.wanted_components) LOG.info("Activating components [%s] (in that order) for action [%s]" % ("->".join(component_order), action)) self._verify_components(component_order, instances) @@ -264,11 +233,5 @@ class ActionRunner(object): self._write_rc_file(root_dir) self._run_instances(action, component_order, instances) - def _setup_root(self, root_dir): - if not sh.isdir(root_dir): - sh.mkdir(root_dir) - - def run(self, persona_fn, root_dir): - persona = self._load_persona(persona_fn) - self._setup_root(root_dir) + def run(self, persona, root_dir): self._run_action(persona, self.action, root_dir) diff --git a/stack b/stack index d8e3afe2..8ded902d 100755 --- a/stack +++ b/stack @@ -28,6 +28,8 @@ from devstack import env from devstack import env_rc from devstack import log as logging from devstack import opts +from devstack import passwords +from devstack import persona from devstack import settings from devstack import shell as sh from devstack import utils @@ -101,6 +103,17 @@ def load_rc_files(): return len(fns) +def load_verify_persona(fn, dist): + instance = persona.Persona.load_file(fn) + instance.verify(dist) + return instance + + +def setup_root(root_dir): + if not sh.isdir(root_dir): + sh.mkdir(root_dir) + + def run(args): action = args.pop("action", '').strip().lower() if not (action in settings.ACTIONS): @@ -117,6 +130,7 @@ def run(args): print(utils.color_text("No root directory specified!", "red")) return False root_dir = sh.abspth(root_dir) + setup_root(root_dir) persona_fn = args.pop('persona_fn') if not persona_fn or not sh.isfile(persona_fn): @@ -137,16 +151,27 @@ def run(args): loaded_rcs = True # Stash the dryrun value (if any) into the global configuration - sh.set_dryrun(args.pop('dryrun')) + sh.set_dryrun(args.get('dryrun', False)) + # Params for the runner... dist = distro.Distro.get_current() + persona_inst = load_verify_persona(persona_fn, dist) config = cfg.get_config() - runner = actions.ActionRunner(dist, action, config, **args) + pw_gen = passwords.PasswordGenerator(config, args.get('prompt_for_passwords', True)) + pkg_cls = dist.get_packager_factory() + pkg_manager = pkg_cls(dist, args.get('keep_old', False)) - LOG.info("Starting action [%s] on %s for distro [%s]" % (action, date.rcf8222date(), dist)) + runner = actions.ActionRunner(dist, action, + config, pw_gen, + pkg_manager, + **args) + + LOG.info("Starting action [%s] on %s for distro: %s" % (action, date.rcf8222date(), dist.name)) + LOG.info("Using persona: %s" % (persona_inst)) + LOG.info("In root directory: %s" % (root_dir)) start_time = time.time() - runner.run(persona_fn, root_dir) + runner.run(persona_inst, root_dir) end_time = time.time() LOG.info("It took (%s) to complete action [%s]" %