x/anvil
Code Issues Proposed changes

Update so that component constructor is simpler + cleanups of various coding parts

Browse Source
This commit is contained in:
Joshua Harlow
2012-06-04 12:34:50 -07:00
parent e74fe46374
commit 4e587fc12f
32 changed files with 488 additions and 446 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
anvil/actions/__init__.py
View File

@@ -28,7 +28,7 @@ _NAMES_TO_RUNNER = {
stop.StopAction.get_action_name(): stop.StopAction,
uninstall.UninstallAction.get_action_name(): uninstall.UninstallAction,
}
_RUNNER_TO_NAMES = dict((v,k) for k, v in _NAMES_TO_RUNNER.items())
_RUNNER_TO_NAMES = dict((v, k) for k, v in _NAMES_TO_RUNNER.items())
def get_action_names():

89
anvil/actions/base.py
View File

@@ -16,14 +16,11 @@
import abc
import collections
import glob
from anvil import colorizer
from anvil import exceptions as excp
from anvil import log as logging
from anvil import packager
from anvil import phase
from anvil import pip
from anvil import shell as sh
from anvil import utils
@@ -45,11 +42,11 @@ class Action(object):
@staticmethod
def get_lookup_name():
return None
raise NotImplementedError()
@staticmethod
def get_action_name():
return None
raise NotImplementedError()
@abc.abstractmethod
def _run(self, persona, component_order, instances):
@@ -82,32 +79,43 @@ class Action(object):
"""
Create component objects for each component in the persona.
"""
components = persona.wanted_components
desired_subsystems = persona.wanted_subsystems or {}
component_opts = persona.component_options or {}
p_subsystems = persona.wanted_subsystems or {}
p_opts = persona.component_options or {}
instances = {}
pip_factory = packager.PackagerFactory(self.distro, pip.Packager)
pkg_factory = packager.PackagerFactory(self.distro, self.distro.get_default_package_manager_cls())
for c in components:
(cls, my_info) = self.distro.extract_component(c, self.get_lookup_name())
for c in persona.wanted_components:
((cls, opts), siblings) = self.distro.extract_component(c, self.get_lookup_name())
LOG.debug("Constructing class %s" % (cls))
cls_kvs = {}
cls_kvs['runner'] = self
cls_kvs.update(self.get_component_dirs(c))
cls_kvs['subsystem_info'] = my_info.get('subsystems') or {}
cls_kvs['all_instances'] = instances
cls_kvs['siblings'] = siblings
# Merge subsystems info with wanted subsystems
sub_systems = opts.get('subsystems') or {}
desired_subs = p_subsystems.get(c) or []
merged_sub_systems = {}
for d in desired_subs:
if d in sub_systems:
merged_sub_systems[d] = sub_systems[d]
else:
merged_sub_systems[d] = {}
cls_kvs['subsystems'] = merged_sub_systems
cls_kvs['instances'] = instances
cls_kvs['name'] = c
cls_kvs['keep_old'] = self.keep_old
cls_kvs['desired_subsystems'] = desired_subsystems.get(c) or set()
cls_kvs['options'] = component_opts.get(c) or {}
cls_kvs['pip_factory'] = pip_factory
cls_kvs['packager_factory'] = pkg_factory
# The above is not overrideable...
for (k, v) in my_info.items():
# Merge options with keep old
options = {}
options['keep_old'] = self.keep_old
options.update(self.get_component_dirs(c))
merge_options = p_opts.get(c) or {}
options.update(merge_options)
# The above is not overrideable... (except for options key)
for (k, v) in opts.items():
if k not in cls_kvs:
cls_kvs[k] = v
elif k == 'options':
options.update(v)
else:
LOG.warn("You can not override component constructor variable named %s.", colorizer.quote(k))
cls_kvs['options'] = options
LOG.debug("Construction of %r params are %s", c, cls_kvs)
instances[c] = cls(**cls_kvs)
return instances
@@ -121,31 +129,33 @@ class Action(object):
for c in component_order:
instances[c].warm_configs()
def _get_phase_dir(self, action_name=None):
if not action_name:
action_name = self.get_action_name()
return sh.joinpths(self.root_dir, "phases", action_name)
def _get_phase_fn(self, phase_name):
dirname = self._get_phase_dir()
sh.mkdirslist(dirname)
return sh.joinpths(dirname, "%s.phases" % (phase_name.lower()))
def _run_phase(self, functors, component_order, instances, phase_name):
"""
Run a given 'functor' across all of the components, in order.
"""
component_results = dict()
phase_recorder = None
if phase_name:
phase_fn = "%s.%s.phases" % (self.get_action_name(), phase_name.lower())
phase_recorder = phase.PhaseRecorder(sh.joinpths(self.root_dir, phase_fn))
phase_recorder = phase.PhaseRecorder(self._get_phase_fn(phase_name))
else:
phase_recorder = phase.NullPhaseRecorder()
for c in component_order:
instance = instances[c]
if phase_recorder and phase_recorder.has_ran(instance.component_name):
if phase_recorder.has_ran(c):
LOG.debug("Skipping phase named %r for component %r since it already happened.", phase_name, c)
else:
try:
result = None
if phase_recorder:
with phase_recorder.mark(instance.component_name):
if functors.start:
functors.start(instance)
if functors.run:
result = functors.run(instance)
if functors.end:
functors.end(instance, result)
else:
with phase_recorder.mark(c):
if functors.start:
functors.start(instance)
if functors.run:
@@ -160,12 +170,9 @@ class Action(object):
raise
return component_results
def _delete_phase_files(self, names):
for n in names:
phases_path = sh.joinpths(self.root_dir, '%s.*.phases' % (n))
for fn in glob.glob(phases_path):
if sh.isfile(fn):
sh.unlink(fn)
def _delete_phase_files(self, action_names):
for n in action_names:
sh.deldir(self._get_phase_dir(n))
def run(self, persona):
instances = self._construct_instances(persona)

15
anvil/actions/install.py
View File

@@ -14,7 +14,6 @@
# License for the specific language governing permissions and limitations
# under the License.
from anvil import colorizer
from anvil import env_rc
from anvil import log
@@ -54,7 +53,7 @@ class InstallAction(base.Action):
self._write_rc_file()
self._run_phase(
PhaseFunctors(
start=lambda i: LOG.info('Downloading %s.', colorizer.quote(i.component_name)),
start=lambda i: LOG.info('Downloading %s.', colorizer.quote(i.name)),
run=lambda i: i.download(),
end=lambda i, result: LOG.info("Performed %s downloads.", result),
),
@@ -64,7 +63,7 @@ class InstallAction(base.Action):
)
self._run_phase(
PhaseFunctors(
start=lambda i: LOG.info('Configuring %s.', colorizer.quote(i.component_name)),
start=lambda i: LOG.info('Configuring %s.', colorizer.quote(i.name)),
run=lambda i: i.configure(),
end=lambda i, result: LOG.info("Configured %s items.", colorizer.quote(result)),
),
@@ -84,19 +83,19 @@ class InstallAction(base.Action):
)
def install_start(instance):
subsystems = set(list(instance.desired_subsystems))
subsystems = set(list(instance.subsystems))
if subsystems:
utils.log_iterable(subsystems, logger=LOG,
header='Installing %s using subsystems' % colorizer.quote(instance.component_name))
header='Installing %s using subsystems' % colorizer.quote(instance.name))
else:
LOG.info("Installing %s.", colorizer.quote(instance.component_name))
LOG.info("Installing %s.", colorizer.quote(instance.name))
self._run_phase(
PhaseFunctors(
start=install_start,
run=lambda i: i.install(),
end=(lambda i, result: LOG.info("Finished install of %s items - check %s for information on what was done.",
colorizer.quote(i.component_name), colorizer.quote(result))),
colorizer.quote(i.name), colorizer.quote(result))),
),
component_order,
instances,
@@ -104,7 +103,7 @@ class InstallAction(base.Action):
)
self._run_phase(
PhaseFunctors(
start=lambda i: LOG.info('Post-installing %s.', colorizer.quote(i.component_name)),
start=lambda i: LOG.info('Post-installing %s.', colorizer.quote(i.name)),
run=lambda i: i.post_install(),
end=None
),

5
anvil/actions/start.py
View File

@@ -14,7 +14,6 @@
# License for the specific language governing permissions and limitations
# under the License.
from anvil import colorizer
from anvil import log
@@ -58,7 +57,7 @@ class StartAction(base.Action):
)
self._run_phase(
PhaseFunctors(
start=lambda i: LOG.info('Starting %s.', i.component_name),
start=lambda i: LOG.info('Starting %s.', i.name),
run=lambda i: i.start(),
end=lambda i, result: LOG.info("Start %s applications", colorizer.quote(result)),
),
@@ -68,7 +67,7 @@ class StartAction(base.Action):
)
self._run_phase(
PhaseFunctors(
start=lambda i: LOG.info('Post-starting %s.', colorizer.quote(i.component_name)),
start=lambda i: LOG.info('Post-starting %s.', colorizer.quote(i.name)),
run=lambda i: i.post_start(),
end=None,
),

22
anvil/actions/status.py
View File

@@ -15,6 +15,7 @@
# under the License.
from anvil import colorizer
from anvil import constants
from anvil import log
from anvil.actions import base
@@ -37,8 +38,25 @@ class StatusAction(base.Action):
def _fetch_status(self, component):
return component.status()
def _print_status(self, component, status):
LOG.info("Status of %s is %s.", colorizer.quote(component.component_name), colorizer.quote(str(status)))
def _quote_status(self, status):
if status == constants.STATUS_UNKNOWN:
return colorizer.quote(status, quote_color='yellow')
elif status == constants.STATUS_STARTED or status == constants.STATUS_INSTALLED:
return colorizer.quote(status, quote_color='green')
else:
return colorizer.quote(status, quote_color='red')
def _print_status(self, component, result):
if isinstance(result, (dict)):
LOG.info("Status of %s is:", colorizer.quote(component.name))
for (name, status) in result.items():
LOG.info("|-- %s --> %s.", colorizer.quote(name, quote_color='blue'), self._quote_status(status))
elif isinstance(result, (list, set)):
LOG.info("Status of %s is:", colorizer.quote(component.name))
for status in result:
LOG.info("|-- %s.", self._quote_status(status))
else:
LOG.info("Status of %s is %s.", colorizer.quote(component.name), self._quote_status(result))
def _run(self, persona, component_order, instances):
self._run_phase(

2
anvil/actions/stop.py
View File

@@ -42,7 +42,7 @@ class StopAction(base.Action):
def _run(self, persona, component_order, instances):
self._run_phase(
PhaseFunctors(
start=lambda i: LOG.info('Stopping %s.', colorizer.quote(i.component_name)),
start=lambda i: LOG.info('Stopping %s.', colorizer.quote(i.name)),
run=lambda i: i.stop(),
end=lambda i, result: LOG.info("Stopped %s items.", colorizer.quote(result)),
),

7
anvil/actions/uninstall.py
View File

@@ -18,7 +18,6 @@ from anvil import colorizer
from anvil import log
from anvil.actions import base
from anvil.actions import install
from anvil.actions.base import PhaseFunctors
@@ -43,7 +42,7 @@ class UninstallAction(base.Action):
def _run(self, persona, component_order, instances):
self._run_phase(
PhaseFunctors(
start=lambda i: LOG.info('Unconfiguring %s.', colorizer.quote(i.component_name)),
start=lambda i: LOG.info('Unconfiguring %s.', colorizer.quote(i.name)),
run=lambda i: i.unconfigure(),
end=None,
),
@@ -63,7 +62,7 @@ class UninstallAction(base.Action):
)
self._run_phase(
PhaseFunctors(
start=lambda i: LOG.info('Uninstalling %s.', colorizer.quote(i.component_name)),
start=lambda i: LOG.info('Uninstalling %s.', colorizer.quote(i.name)),
run=lambda i: i.uninstall(),
end=None,
),
@@ -73,7 +72,7 @@ class UninstallAction(base.Action):
)
self._run_phase(
PhaseFunctors(
start=lambda i: LOG.info('Post-uninstalling %s.', colorizer.quote(i.component_name)),
start=lambda i: LOG.info('Post-uninstalling %s.', colorizer.quote(i.name)),
run=lambda i: i.post_uninstall(),
end=None,
),

3
anvil/cfg_helpers.py
View File

@@ -14,6 +14,7 @@
# License for the specific language governing permissions and limitations
# under the License.
from anvil import constants
from anvil import log as logging
from anvil import settings
from anvil import shell as sh
@@ -77,7 +78,7 @@ def get_config_locations(start_locations=None):
if start_locations:
locs.extend(start_locations)
locs.append(settings.CONFIG_LOCATION)
locs.append(sh.joinpths("/etc", settings.PROG_NAME, settings.CONFIG_NAME))
locs.append(sh.joinpths("/etc", constants.PROG_NAME, settings.CONFIG_NAME))
return locs

228
anvil/component.py
View File

@@ -14,52 +14,46 @@
# License for the specific language governing permissions and limitations
# under the License.
import copy
import re
import weakref
from anvil import colorizer
from anvil import constants
from anvil import downloader as down
from anvil import exceptions as excp
from anvil import importer
from anvil import log as logging
from anvil import packager
from anvil import pip
from anvil import shell as sh
from anvil import trace as tr
from anvil import utils
LOG = logging.getLogger(__name__)
# Runtime status constants (return by runtime status)
# TODO: move...
STATUS_UNKNOWN = "unknown"
STATUS_STARTED = "started"
STATUS_STOPPED = "stopped"
# Progress bar titles
UNINSTALL_TITLE = 'Uninstalling'
INSTALL_TITLE = 'Installing'
class ComponentBase(object):
def __init__(self,
desired_subsystems,
subsystem_info,
subsystems,
runner,
component_dir,
trace_dir,
app_dir,
cfg_dir,
all_instances,
instances,
options,
name,
siblings,
*args,
**kargs):
self.desired_subsystems = desired_subsystems
self.instances = all_instances
self.component_name = name
self.subsystem_info = subsystem_info
self.subsystems = subsystems
self.name = name
self.options = options
# All the other active instances
self.instances = instances
# All the other class names that can be used alongside this class
self.siblings = siblings
# The runner has a reference to us, so use a weakref here to
# avoid breaking garbage collection.
self.runner = weakref.proxy(runner)
@@ -68,49 +62,39 @@ class ComponentBase(object):
self.cfg = runner.cfg
self.distro = runner.distro
# Required component directories
self.component_dir = component_dir
self.trace_dir = trace_dir
self.app_dir = app_dir
self.cfg_dir = cfg_dir
def get_option(self, opt_name, def_val=None):
val = self.options.get(opt_name, def_val)
LOG.debug("Fetched option %r with value %r for component: %s", opt_name, val, self)
return val
def verify(self):
# Ensure subsystems are known...
knowns = self.known_subsystems()
for s in self.desired_subsystems:
for s in self.subsystems:
if s not in knowns:
raise ValueError("Unknown subsystem %r requested for (%s)" % (s, self))
for s in self.subsystem_info.keys():
if s not in knowns:
raise ValueError("Unknown subsystem %r provided for (%s)" % (s, self))
known_options = self.known_options()
for s in self.options:
if s not in known_options:
LOG.warning("Unknown option %r provided for (%s)" % (s, self))
raise ValueError("Unknown subsystem %r requested for component: %s" % (s, self))
def __str__(self):
return "%s: %s" % (self.__class__.__name__, self.component_name)
return "%s@%s" % (self.__class__.__name__, self.name)
def _get_params(self):
return {
'APP_DIR': self.app_dir,
'COMPONENT_DIR': self.component_dir,
'CONFIG_DIR': self.cfg_dir,
'TRACE_DIR': self.trace_dir,
'APP_DIR': self.get_option('app_dir'),
'COMPONENT_DIR': self.get_option('component_dir'),
'CONFIG_DIR': self.get_option('cfg_dir'),
'TRACE_DIR': self.get_option('trace_dir'),
}
def _get_trace_files(self):
trace_dir = self.get_option('trace_dir')
return {
'install': tr.trace_fn(self.trace_dir, "install"),
'start': tr.trace_fn(self.trace_dir, "start"),
'install': tr.trace_fn(trace_dir, "install"),
'start': tr.trace_fn(trace_dir, "start"),
}
def known_subsystems(self):
return set()
def known_options(self):
return set()
def warm_configs(self):
pass
@@ -122,11 +106,11 @@ class ComponentBase(object):
class PkgInstallComponent(ComponentBase):
def __init__(self, packager_factory, *args, **kargs):
def __init__(self, *args, **kargs):
ComponentBase.__init__(self, *args, **kargs)
self.tracewriter = tr.TraceWriter(self._get_trace_files()['install'], break_if_there=False)
self.packages = kargs.get('packages', list())
self.packager_factory = packager_factory
self.packager_factory = packager.PackagerFactory(self.distro, self.distro.get_default_package_manager_cls())
def _get_download_locations(self):
return list()
@@ -139,7 +123,7 @@ class PkgInstallComponent(ComponentBase):
if not uri:
raise ValueError(("Could not find uri in config to download "
"from at section %s for option %s") % (section, key))
target_directory = self.app_dir
target_directory = self.get_option('app_dir')
if 'subdir' in info:
target_directory = sh.joinpths(target_directory, info["subdir"])
branch = None
@@ -185,26 +169,26 @@ class PkgInstallComponent(ComponentBase):
return mp
def _get_packages(self):
pkg_list = list(self.packages)
for name in self.desired_subsystems:
if name in self.subsystem_info:
LOG.debug("Extending package list with packages for subsystem %r", name)
pkg_list.extend(self.subsystem_info[name].get('packages', []))
pkg_list = copy.deepcopy(self.packages)
for name, values in self.subsystems.items():
if 'packages' in values:
LOG.debug("Extending package list with packages for subsystem: %r", name)
pkg_list.extend(values.get('packages') or [])
return pkg_list
def install(self):
LOG.debug('Preparing to install packages for %r', self.component_name)
LOG.debug('Preparing to install packages for: %r', self.name)
pkgs = self._get_packages()
if pkgs:
pkg_names = [p['name'] for p in pkgs]
utils.log_iterable(pkg_names, logger=LOG,
header="Setting up %s distribution packages" % (len(pkg_names)))
with utils.progress_bar(INSTALL_TITLE, len(pkgs)) as p_bar:
with utils.progress_bar('Installing', len(pkgs)) as p_bar:
for (i, p) in enumerate(pkgs):
self.tracewriter.package_installed(p)
self.packager_factory.get_packager_for(p).install(p)
p_bar.update(i + 1)
return self.trace_dir
return self.get_option('trace_dir')
def pre_install(self):
pkgs = self._get_packages()
@@ -223,14 +207,14 @@ class PkgInstallComponent(ComponentBase):
return contents
def _get_target_config_name(self, config_fn):
return sh.joinpths(self.cfg_dir, config_fn)
cfg_dir = self.get_option('cfg_dir')
return sh.joinpths(cfg_dir, config_fn)
def _get_source_config(self, config_fn):
return utils.load_template(self.component_name, config_fn)
return utils.load_template(self.name, config_fn)
def _get_link_dir(self):
root_link_dir = self.distro.get_command_config('base_link_dir')
return sh.joinpths(root_link_dir, self.component_name)
return sh.joinpths(self.distro.get_command_config('base_link_dir'), self.name)
def _get_symlinks(self):
links = dict()
@@ -284,24 +268,23 @@ class PkgInstallComponent(ComponentBase):
class PythonInstallComponent(PkgInstallComponent):
def __init__(self, pip_factory, *args, **kargs):
def __init__(self, *args, **kargs):
PkgInstallComponent.__init__(self, *args, **kargs)
self.pips = kargs.get('pips', list())
self.pip_factory = pip_factory
self.pip_factory = packager.PackagerFactory(self.distro, pip.Packager)
def _get_python_directories(self):
py_dirs = {
self.component_name: self.app_dir,
self.name: self.get_option('app_dir'),
}
return py_dirs
def _get_pips(self):
pip_list = list(self.pips)
for name in self.desired_subsystems:
if name in self.subsystem_info:
LOG.debug("Extending pip list with pips for subsystem %r" % (name))
subsystem_pips = self.subsystem_info[name].get('pips', list())
pip_list.extend(subsystem_pips)
pip_list = copy.deepcopy(self.pips)
for name, values in self.subsystems.items():
if 'pips' in values:
LOG.debug("Extending pip list with pips for subsystem: %r" % (name))
pip_list.extend(values.get('pips') or [])
return pip_list
def _install_pips(self):
@@ -310,7 +293,7 @@ class PythonInstallComponent(PkgInstallComponent):
pip_names = [p['name'] for p in pips]
utils.log_iterable(pip_names, logger=LOG,
header="Setting up %s python packages" % (len(pip_names)))
with utils.progress_bar(INSTALL_TITLE, len(pips)) as p_bar:
with utils.progress_bar('Installing', len(pips)) as p_bar:
for (i, p) in enumerate(pips):
self.tracewriter.pip_installed(p)
self.pip_factory.get_packager_for(p).install(p)
@@ -333,7 +316,9 @@ class PythonInstallComponent(PkgInstallComponent):
if py_dirs:
real_dirs = dict()
for (name, wkdir) in py_dirs.items():
real_dirs[name] = wkdir or self.app_dir
real_dirs[name] = wkdir
if not real_dirs[name]:
real_dirs[name] = self.get_option('app_dir')
utils.log_iterable(real_dirs.values(), logger=LOG,
header="Setting up %s python directories" % (len(real_dirs)))
setup_cmd = self.distro.get_command('python', 'setup')
@@ -344,7 +329,7 @@ class PythonInstallComponent(PkgInstallComponent):
cwd=working_dir,
run_as_root=True)
py_trace_name = "%s.%s" % (name, 'python')
py_writer = tr.TraceWriter(tr.trace_fn(self.trace_dir,
py_writer = tr.TraceWriter(tr.trace_fn(self.get_option('trace_dir'),
py_trace_name), break_if_there=False)
# Format or json encoding isn't really needed here since this is
# more just for information output/lookup if desired.
@@ -364,11 +349,10 @@ class PythonInstallComponent(PkgInstallComponent):
class PkgUninstallComponent(ComponentBase):
def __init__(self, packager_factory, *args, **kargs):
def __init__(self, *args, **kargs):
ComponentBase.__init__(self, *args, **kargs)
self.tracereader = tr.TraceReader(self._get_trace_files()['install'])
self.keep_old = kargs.get('keep_old', False)
self.packager_factory = packager_factory
self.packager_factory = packager.PackagerFactory(self.distro, self.distro.get_default_package_manager_cls())
def unconfigure(self):
self._unconfigure_files()
@@ -404,7 +388,7 @@ class PkgUninstallComponent(ComponentBase):
pass
def _uninstall_pkgs(self):
if self.keep_old:
if self.get_option('keep_old', False):
LOG.info('Keep-old flag set, not removing any packages.')
else:
pkgs = self.tracereader.packages_installed()
@@ -413,7 +397,7 @@ class PkgUninstallComponent(ComponentBase):
utils.log_iterable(pkg_names, logger=LOG,
header="Potentially removing %s packages" % (len(pkg_names)))
which_removed = set()
with utils.progress_bar(UNINSTALL_TITLE, len(pkgs), reverse=True) as p_bar:
with utils.progress_bar('Uninstalling', len(pkgs), reverse=True) as p_bar:
for (i, p) in enumerate(pkgs):
if self.packager_factory.get_packager_for(p).remove(p):
which_removed.add(p['name'])
@@ -433,7 +417,7 @@ class PkgUninstallComponent(ComponentBase):
dirs_made = self.tracereader.dirs_made()
if dirs_made:
dirs_made = [sh.abspth(d) for d in dirs_made]
if self.keep_old:
if self.get_option('keep_old', False):
download_places = [path_location[0] for path_location in self.tracereader.download_locations()]
if download_places:
utils.log_iterable(download_places, logger=LOG,
@@ -451,9 +435,9 @@ class PkgUninstallComponent(ComponentBase):
class PythonUninstallComponent(PkgUninstallComponent):
def __init__(self, pip_factory, *args, **kargs):
def __init__(self, *args, **kargs):
PkgUninstallComponent.__init__(self, *args, **kargs)
self.pip_factory = pip_factory
self.pip_factory = packager.PackagerFactory(self.distro, pip.Packager)
def uninstall(self):
self._uninstall_python()
@@ -461,7 +445,7 @@ class PythonUninstallComponent(PkgUninstallComponent):
PkgUninstallComponent.uninstall(self)
def _uninstall_pips(self):
if self.keep_old:
if self.get_option('keep_old', False):
LOG.info('Keep-old flag set, not removing any python packages.')
else:
pips = self.tracereader.pips_installed()
@@ -469,7 +453,7 @@ class PythonUninstallComponent(PkgUninstallComponent):
pip_names = set([p['name'] for p in pips])
utils.log_iterable(pip_names, logger=LOG,
header="Uninstalling %s python packages" % (len(pip_names)))
with utils.progress_bar(UNINSTALL_TITLE, len(pips), reverse=True) as p_bar:
with utils.progress_bar('Uninstalling', len(pips), reverse=True) as p_bar:
for (i, p) in enumerate(pips):
try:
self.pip_factory.get_packager_for(p).remove(p)
@@ -534,16 +518,16 @@ class ProgramRuntime(ComponentBase):
am_configured = 0
if not apps_to_start:
return am_configured
# First make a pass and make sure all runtime
# First make a pass and make sure all runtime
# (e.g. upstart starting)
# config files are in place....
run_type = self._fetch_run_type()
cls = importer.import_entry_point(run_type)
instance = cls(self.cfg, self.component_name, self.trace_dir)
instance = cls(self.cfg, self.name, self.get_option('trace_dir'))
for app_info in apps_to_start:
app_name = app_info["name"]
app_pth = app_info.get("path", app_name)
app_dir = app_info.get("app_dir", self.app_dir)
app_dir = app_info.get("app_dir", self.get_option('app_dir'))
# Configure it with the given settings
LOG.debug("Configuring runner %r for program %r", run_type, app_name)
cfg_am = instance.configure(app_name,
@@ -562,11 +546,11 @@ class ProgramRuntime(ComponentBase):
# Select how we are going to start it
run_type = self._fetch_run_type()
cls = importer.import_entry_point(run_type)
instance = cls(self.cfg, self.component_name, self.trace_dir)
instance = cls(self.cfg, self.name, self.get_option('trace_dir'))
for app_info in apps_to_start:
app_name = app_info["name"]
app_pth = app_info.get("path", app_name)
app_dir = app_info.get("app_dir", self.app_dir)
app_dir = app_info.get("app_dir", self.get_option('app_dir'))
# Adjust the program options now that we have real locations
program_opts = utils.param_replace_list(self._get_app_options(app_name), self._get_param_map(app_name))
# Start it with the given settings
@@ -579,27 +563,25 @@ class ProgramRuntime(ComponentBase):
am_started += 1
return am_started
def _locate_killers(self, apps_started):
killer_instances = dict()
to_kill = list()
def _locate_investigators(self, apps_started):
investigators = dict()
to_investigate = list()
for (app_name, trace_fn, how) in apps_started:
killcls = None
inv_cls = None
try:
killcls = importer.import_entry_point(how)
LOG.debug("Stopping %r using %r", app_name, how)
inv_cls = importer.import_entry_point(how)
except RuntimeError as e:
LOG.warn("Could not load class %s which should be used to stop %s: %s",
colorizer.quote(how), colorizer.quote(app_name), e)
LOG.warn("Could not load class %s which should be used to investigate %s: %s",
colorizer.quote(how), colorizer.quote(app_name), e)
continue
if killcls in killer_instances:
killer = killer_instances[killcls]
investigator = None
if inv_cls in investigators:
investigator = investigators[inv_cls]
else:
killer = killcls(self.cfg,
self.component_name,
self.trace_dir)
killer_instances[killcls] = killer
to_kill.append((app_name, killer))
return to_kill
investigator = inv_cls(self.cfg, self.name, self.get_option('trace_dir'))
investigators[inv_cls] = investigator
to_investigate.append((app_name, investigator))
return to_investigate
def stop(self):
# Anything to stop??
@@ -608,18 +590,48 @@ class ProgramRuntime(ComponentBase):
if not apps_started:
return killed_am
self.pre_stop(apps_started)
to_kill = self._locate_killers(apps_started)
for (app_name, killer) in to_kill:
killer.stop(app_name)
killer.unconfigure()
to_kill = self._locate_investigators(apps_started)
for (app_name, handler) in to_kill:
handler.stop(app_name)
handler.unconfigure()
killed_am += 1
self.post_stop(apps_started)
if len(apps_started) == killed_am:
sh.unlink(self.tracereader.filename())
return killed_am
def _multi_status(self):
try:
apps_started = self.tracereader.apps_started()
except excp.NoTraceException:
return None
if not apps_started:
return None
else:
to_check = self._locate_investigators(apps_started)
results = dict()
for (name, handler) in to_check:
try:
results[name] = handler.status(name)
except AttributeError:
pass # Not all handlers can implement this..
return results
def _status(self):
return constants.STATUS_UNKNOWN
def status(self):
return STATUS_UNKNOWN
stat = self._multi_status()
if not stat:
stat = self._status()
if not stat or stat == constants.STATUS_UNKNOWN:
if self.is_installed():
stat = constants.STATUS_INSTALLED
elif self.is_started():
stat = constants.STATUS_STARTED
else:
stat = constants.STATUS_UNKNOWN
return stat
def restart(self):
return 0

46
anvil/components/db.py
View File

@@ -16,7 +16,9 @@
from anvil import colorizer
from anvil import component as comp
from anvil import constants
from anvil import exceptions as excp
from anvil import importer
from anvil import log as logging
from anvil import shell as sh
from anvil import utils
@@ -49,10 +51,11 @@ class DBUninstaller(comp.PkgUninstallComponent):
def __init__(self, *args, **kargs):
comp.PkgUninstallComponent.__init__(self, *args, **kargs)
(runtime_cls, _) = self.distro.extract_component(self.component_name, 'running')
if not runtime_cls:
runtime_cls_name = self.siblings.get('running')
if not runtime_cls_name:
self.runtime = DBRuntime(*args, **kargs)
else:
runtime_cls = importer.import_entry_point(runtime_cls_name)
self.runtime = runtime_cls(*args, **kargs)
def warm_configs(self):
@@ -89,10 +92,11 @@ class DBInstaller(comp.PkgInstallComponent):
def __init__(self, *args, **kargs):
comp.PkgInstallComponent.__init__(self, *args, **kargs)
(runtime_cls, _) = self.distro.extract_component(self.component_name, 'running')
if not runtime_cls:
runtime_cls_name = self.siblings.get('running')
if not runtime_cls_name:
self.runtime = DBRuntime(*args, **kargs)
else:
runtime_cls = importer.import_entry_point(runtime_cls_name)
self.runtime = runtime_cls(*args, **kargs)
def _get_param_map(self, config_fn):
@@ -161,16 +165,13 @@ class DBRuntime(comp.EmptyRuntime):
dbtype = self.cfg.get("db", "type")
distro_options = self.distro.get_command_config(dbtype)
if distro_options is None:
msg = BASE_ERROR % (act, dbtype)
raise NotImplementedError(msg)
raise NotImplementedError(BASE_ERROR % (act, dbtype))
return self.distro.get_command(dbtype, act)
def start(self):
if self.status() != comp.STATUS_STARTED:
if self._status() != constants.STATUS_STARTED:
startcmd = self._get_run_actions('start', excp.StartException)
sh.execute(*startcmd,
run_as_root=True,
check_exit_code=True)
sh.execute(*startcmd, run_as_root=True, check_exit_code=True)
LOG.info("Please wait %s seconds while it starts up." % self.wait_time)
sh.sleep(self.wait_time)
return 1
@@ -178,11 +179,9 @@ class DBRuntime(comp.EmptyRuntime):
return 0
def stop(self):
if self.status() != comp.STATUS_STOPPED:
if self._status() != constants.STATUS_STOPPED:
stopcmd = self._get_run_actions('stop', excp.StopException)
sh.execute(*stopcmd,
run_as_root=True,
check_exit_code=True)
sh.execute(*stopcmd, run_as_root=True, check_exit_code=True)
return 1
else:
return 0
@@ -190,26 +189,19 @@ class DBRuntime(comp.EmptyRuntime):
def restart(self):
LOG.info("Restarting your database.")
restartcmd = self._get_run_actions('restart', excp.RestartException)
sh.execute(*restartcmd,
run_as_root=True,
check_exit_code=True)
sh.execute(*restartcmd, run_as_root=True, check_exit_code=True)
LOG.info("Please wait %s seconds while it restarts." % self.wait_time)
sh.sleep(self.wait_time)
return 1
def status(self):
def _status(self):
statuscmd = self._get_run_actions('status', excp.StatusException)
run_result = sh.execute(*statuscmd,
run_as_root=True,
check_exit_code=False)
if not run_result:
return comp.STATUS_UNKNOWN
(sysout, stderr) = run_result
(sysout, stderr) = sh.execute(*statuscmd, run_as_root=True, check_exit_code=False)
combined = (str(sysout) + str(stderr)).lower()
if combined.find("running") != -1:
return comp.STATUS_STARTED
return constants.STATUS_STARTED
elif combined.find("stop") != -1 or \
combined.find('unrecognized') != -1:
return comp.STATUS_STOPPED
return constants.STATUS_STOPPED
else:
return comp.STATUS_UNKNOWN
return constants.STATUS_UNKNOWN

24
anvil/components/glance.py
View File

@@ -96,7 +96,7 @@ class GlanceInstaller(GlanceMixin, comp.PythonInstallComponent):
def pre_install(self):
comp.PythonInstallComponent.pre_install(self)
if self.cfg.getboolean('glance', 'eliminate_pip_gits'):
fn = sh.joinpths(self.app_dir, 'tools', 'pip-requires')
fn = sh.joinpths(self.get_option('app_dir'), 'tools', 'pip-requires')
if sh.isfile(fn):
new_lines = []
for line in sh.load_file(fn).splitlines():
@@ -118,7 +118,7 @@ class GlanceInstaller(GlanceMixin, comp.PythonInstallComponent):
real_fn = config_fn
if config_fn == LOGGING_CONF:
real_fn = LOGGING_SOURCE_FN
fn = sh.joinpths(self.app_dir, 'etc', real_fn)
fn = sh.joinpths(self.get_option('app_dir'), 'etc', real_fn)
return (fn, sh.load_file(fn))
def _config_adjust_registry(self, contents, fn):
@@ -160,7 +160,7 @@ class GlanceInstaller(GlanceMixin, comp.PythonInstallComponent):
with io.BytesIO(contents) as stream:
config = cfg.RewritableConfigParser()
config.readfp(stream)
img_store_dir = self._get_image_dir()
img_store_dir = sh.joinpths(self.get_option('component_dir'), 'images')
config.set('DEFAULT', 'debug', True)
config.set('DEFAULT', 'verbose', True)
config.set('DEFAULT', 'default_store', 'file')
@@ -203,15 +203,11 @@ class GlanceInstaller(GlanceMixin, comp.PythonInstallComponent):
else:
return contents
def _get_image_dir(self):
# This might be changed often so make it a function
return sh.joinpths(self.component_dir, 'images')
class GlanceRuntime(GlanceMixin, comp.PythonRuntime):
def __init__(self, *args, **kargs):
comp.PythonRuntime.__init__(self, *args, **kargs)
self.bin_dir = sh.joinpths(self.app_dir, BIN_DIR)
self.bin_dir = sh.joinpths(self.get_option('app_dir'), BIN_DIR)
self.wait_time = max(self.cfg.getint('DEFAULT', 'service_wait_seconds'), 1)
self.do_upload = True
if 'no-load-images' in self.options:
@@ -219,11 +215,13 @@ class GlanceRuntime(GlanceMixin, comp.PythonRuntime):
def _get_apps_to_start(self):
apps = list()
for subsys in self.desired_subsystems:
apps.append({
'name': SUB_TO_APP[subsys],
'path': sh.joinpths(self.bin_dir, SUB_TO_APP[subsys]),
})
for name, values in self.subsystems.items():
if name in SUB_TO_APP:
subsys = name
apps.append({
'name': SUB_TO_APP[subsys],
'path': sh.joinpths(self.bin_dir, SUB_TO_APP[subsys]),
})
return apps
def _get_app_options(self, app):

78
anvil/components/horizon.py
View File

@@ -16,10 +16,10 @@
from anvil import colorizer
from anvil import component as comp
from anvil import constants
from anvil import exceptions as excp
from anvil import log as logging
from anvil import shell as sh
from anvil import utils
from anvil.helpers import db as dbhelper
@@ -68,9 +68,7 @@ class HorizonUninstaller(comp.PythonUninstallComponent):
class HorizonInstaller(comp.PythonInstallComponent):
def __init__(self, *args, **kargs):
comp.PythonInstallComponent.__init__(self, *args, **kargs)
self.horizon_dir = sh.joinpths(self.app_dir, ROOT_HORIZON)
self.dash_dir = sh.joinpths(self.app_dir, ROOT_DASH)
self.log_dir = sh.joinpths(self.component_dir, LOGS_DIR)
self.log_dir = sh.joinpths(self.get_option('component_dir'), LOGS_DIR)
def _get_download_locations(self):
places = list()
@@ -111,7 +109,8 @@ class HorizonInstaller(comp.PythonInstallComponent):
def _get_target_config_name(self, config_name):
if config_name == HORIZON_PY_CONF:
# FIXME don't write to checked out locations...
return sh.joinpths(self.dash_dir, *HORIZON_PY_CONF_TGT)
dash_dir = sh.joinpths(self.get_option('app_dir'), ROOT_DASH)
return sh.joinpths(dash_dir, *HORIZON_PY_CONF_TGT)
else:
return comp.PythonInstallComponent._get_target_config_name(self, config_name)
@@ -120,13 +119,13 @@ class HorizonInstaller(comp.PythonInstallComponent):
def _setup_blackhole(self):
# Create an empty directory that apache uses as docroot
self.tracewriter.dirs_made(*sh.mkdirslist(sh.joinpths(self.app_dir, BLACKHOLE_DIR)))
self.tracewriter.dirs_made(*sh.mkdirslist(sh.joinpths(self.get_option('app_dir'), BLACKHOLE_DIR)))
def _sync_db(self):
# Initialize the horizon database (it stores sessions and notices shown to users).
# The user system is external (keystone).
LOG.info("Syncing horizon to database: %s", colorizer.quote(DB_NAME))
sh.execute(*DB_SYNC_CMD, cwd=self.app_dir)
sh.execute(*DB_SYNC_CMD, cwd=self.get_option('app_dir'))
def _setup_db(self):
dbhelper.drop_db(self.cfg, self.distro, DB_NAME)
@@ -136,7 +135,7 @@ class HorizonInstaller(comp.PythonInstallComponent):
comp.PythonInstallComponent.pre_install(self)
self.tracewriter.dirs_made(*sh.mkdirslist(self.log_dir))
if self.cfg.getboolean('horizon', 'eliminate_pip_gits'):
fn = sh.joinpths(self.app_dir, 'tools', 'pip-requires')
fn = sh.joinpths(self.get_option('app_dir'), 'tools', 'pip-requires')
if sh.isfile(fn):
new_lines = []
for line in sh.load_file(fn).splitlines():
@@ -154,7 +153,6 @@ class HorizonInstaller(comp.PythonInstallComponent):
self._setup_db()
self._sync_db()
self._setup_blackhole()
# Anything to fixup after it was installed??
self._config_fixups()
def _get_apache_user_group(self):
@@ -172,9 +170,9 @@ class HorizonInstaller(comp.PythonInstallComponent):
mp['USER'] = user
mp['ACCESS_LOG'] = sh.joinpths(self.log_dir, APACHE_ACCESS_LOG_FN)
mp['ERROR_LOG'] = sh.joinpths(self.log_dir, APACHE_ERROR_LOG_FN)
mp['HORIZON_DIR'] = self.app_dir
mp['HORIZON_DIR'] = self.get_option('app_dir')
mp['HORIZON_PORT'] = self.cfg.getdefaulted('horizon', 'port', APACHE_DEF_PORT)
mp['VPN_DIR'] = sh.joinpths(self.app_dir, "vpn")
mp['VPN_DIR'] = sh.joinpths(self.get_option('app_dir'), "vpn")
else:
mp['OPENSTACK_HOST'] = self.cfg.get('host', 'ip')
mp['DB_NAME'] = DB_NAME
@@ -190,57 +188,35 @@ class HorizonRuntime(comp.EmptyRuntime):
comp.EmptyRuntime.__init__(self, *args, **kargs)
def start(self):
curr_status = self.status()
if curr_status == comp.STATUS_STARTED:
return self.restart()
else:
cmds = [{
'cmd': self.distro.get_command('apache', 'start'),
'run_as_root': True,
}]
utils.execute_template(*cmds,
check_exit_code=True,
params={})
if self._status() != constants.STATUS_STARTED:
start_cmd = self.distro.get_command('apache', 'start')
sh.execute(*start_cmd, run_as_root=True, check_exit_code=True)
return 1
else:
return 0
def restart(self):
cmds = [{
'cmd': self.distro.get_command('apache', 'restart'),
'run_as_root': True,
}]
utils.execute_template(*cmds,
check_exit_code=True,
params={})
restart_cmd = self.distro.get_command('apache', 'restart')
sh.execute(*restart_cmd, run_as_root=True, check_exit_code=True)
return 1
def stop(self):
curr_status = self.status()
if curr_status != comp.STATUS_STOPPED:
cmds = [{
'cmd': self.distro.get_command('apache', 'stop'),
'run_as_root': True,
}]
utils.execute_template(*cmds,
check_exit_code=True)
if self._status() != constants.STATUS_STOPPED:
stop_cmd = self.distro.get_command('apache', 'stop')
sh.execute(*stop_cmd, run_as_root=True, check_exit_code=True)
return 1
return 0
else:
return 0
def status(self):
cmds = [{
'cmd': self.distro.get_command('apache', 'status'),
'run_as_root': True,
}]
run_result = utils.execute_template(*cmds,
check_exit_code=False)
if not run_result or not run_result[0]:
return comp.STATUS_UNKNOWN
(sysout, stderr) = run_result[0]
def _status(self):
status_cmd = self.distro.get_command('apache', 'status')
(sysout, stderr) = sh.execute(*status_cmd, run_as_root=True, check_exit_code=False)
combined = (str(sysout) + str(stderr)).lower()
if combined.find("is running") != -1:
return comp.STATUS_STARTED
return constants.STATUS_STARTED
elif combined.find("not running") != -1 or \
combined.find("stopped") != -1 or \
combined.find('unrecognized') != -1:
return comp.STATUS_STOPPED
return constants.STATUS_STOPPED
else:
return comp.STATUS_UNKNOWN
return constants.STATUS_UNKNOWN

19
anvil/components/keystone.py
View File

@@ -21,7 +21,6 @@ import yaml
from anvil import cfg
from anvil import colorizer
from anvil import component as comp
from anvil import date
from anvil import log as logging
from anvil import shell as sh
from anvil import utils
@@ -83,7 +82,7 @@ class KeystoneUninstaller(comp.PythonUninstallComponent):
class KeystoneInstaller(comp.PythonInstallComponent):
def __init__(self, *args, **kargs):
comp.PythonInstallComponent.__init__(self, *args, **kargs)
self.bin_dir = sh.joinpths(self.app_dir, BIN_DIR)
self.bin_dir = sh.joinpths(self.get_option('app_dir'), BIN_DIR)
def _get_download_locations(self):
places = list()
@@ -98,13 +97,10 @@ class KeystoneInstaller(comp.PythonInstallComponent):
self._setup_db()
self._sync_db()
def known_options(self):
return set(['swift', 'quantum'])
def _sync_db(self):
LOG.info("Syncing keystone to database: %s", colorizer.quote(DB_NAME))
mp = self._get_param_map(None)
cmds = [{'cmd': SYNC_DB_CMD}]
cmds = [{'cmd': SYNC_DB_CMD, 'run_as_root': True}]
utils.execute_template(*cmds, cwd=self.bin_dir, params=mp)
def _get_config_files(self):
@@ -120,7 +116,7 @@ class KeystoneInstaller(comp.PythonInstallComponent):
real_fn = LOGGING_SOURCE_FN
elif config_fn == ROOT_CONF:
real_fn = ROOT_SOURCE_FN
fn = sh.joinpths(self.app_dir, 'etc', real_fn)
fn = sh.joinpths(self.get_option('app_dir'), 'etc', real_fn)
return (fn, sh.load_file(fn))
def _config_adjust_logging(self, contents, fn):
@@ -175,17 +171,18 @@ class KeystoneInstaller(comp.PythonInstallComponent):
# params with actual values
mp = comp.PythonInstallComponent._get_param_map(self, config_fn)
mp['BIN_DIR'] = self.bin_dir
mp['CONFIG_FILE'] = sh.joinpths(self.cfg_dir, ROOT_CONF)
mp['CONFIG_FILE'] = sh.joinpths(self.get_option('cfg_dir'), ROOT_CONF)
return mp
class KeystoneRuntime(comp.PythonRuntime):
def __init__(self, *args, **kargs):
comp.PythonRuntime.__init__(self, *args, **kargs)
self.bin_dir = sh.joinpths(self.app_dir, BIN_DIR)
self.bin_dir = sh.joinpths(self.get_option('app_dir'), BIN_DIR)
self.wait_time = max(self.cfg.getint('DEFAULT', 'service_wait_seconds'), 1)
self.init_fn = sh.joinpths(self.trace_dir, INIT_WHAT_HAPPENED)
self.init_what = yaml.load(utils.load_template(self.component_name, INIT_WHAT_FN)[1])
self.init_fn = sh.joinpths(self.get_option('trace_dir'), INIT_WHAT_HAPPENED)
(fn, contents) = utils.load_template(self.name, INIT_WHAT_FN)
self.init_what = yaml.load(contents)
def post_start(self):
if not sh.isfile(self.init_fn):

12
anvil/components/melange.py
View File

@@ -65,7 +65,7 @@ class MelangeUninstaller(comp.PythonUninstallComponent):
class MelangeInstaller(comp.PythonInstallComponent):
def __init__(self, *args, **kargs):
comp.PythonInstallComponent.__init__(self, *args, **kargs)
self.bin_dir = sh.joinpths(self.app_dir, BIN_DIR)
self.bin_dir = sh.joinpths(self.get_option('app_dir'), BIN_DIR)
def _get_download_locations(self):
places = list()
@@ -90,7 +90,7 @@ class MelangeInstaller(comp.PythonInstallComponent):
def _get_param_map(self, config_fn):
mp = comp.PythonInstallComponent._get_param_map(self, config_fn)
mp['CFG_FILE'] = sh.joinpths(self.cfg_dir, ROOT_CONF_REAL_NAME)
mp['CFG_FILE'] = sh.joinpths(self.get_option('cfg_dir'), ROOT_CONF_REAL_NAME)
mp['BIN_DIR'] = self.bin_dir
return mp
@@ -111,14 +111,14 @@ class MelangeInstaller(comp.PythonInstallComponent):
def _get_source_config(self, config_fn):
if config_fn == ROOT_CONF:
# FIXME, maybe we shouldn't be sucking this from the checkout??
fn = sh.joinpths(self.app_dir, 'etc', 'melange', config_fn)
fn = sh.joinpths(self.get_option('app_dir'), 'etc', 'melange', config_fn)
contents = sh.load_file(fn)
return (fn, contents)
return comp.PythonInstallComponent._get_source_config(self, config_fn)
def _get_target_config_name(self, config_fn):
if config_fn == ROOT_CONF:
return sh.joinpths(self.cfg_dir, ROOT_CONF_REAL_NAME)
return sh.joinpths(self.get_option('cfg_dir'), ROOT_CONF_REAL_NAME)
else:
return comp.PythonInstallComponent._get_target_config_name(self, config_fn)
@@ -126,7 +126,7 @@ class MelangeInstaller(comp.PythonInstallComponent):
class MelangeRuntime(comp.PythonRuntime):
def __init__(self, *args, **kargs):
comp.PythonRuntime.__init__(self, *args, **kargs)
self.bin_dir = sh.joinpths(self.app_dir, BIN_DIR)
self.bin_dir = sh.joinpths(self.get_option('app_dir'), BIN_DIR)
self.wait_time = max(self.cfg.getint('DEFAULT', 'service_wait_seconds'), 1)
def _get_apps_to_start(self):
@@ -143,7 +143,7 @@ class MelangeRuntime(comp.PythonRuntime):
def _get_param_map(self, app_name):
mp = comp.PythonRuntime._get_param_map(self, app_name)
mp['CFG_FILE'] = sh.joinpths(self.cfg_dir, ROOT_CONF_REAL_NAME)
mp['CFG_FILE'] = sh.joinpths(self.get_option('cfg_dir'), ROOT_CONF_REAL_NAME)
return mp
def known_options(self):

76
anvil/components/nova.py
View File

@@ -19,7 +19,6 @@ import io
from anvil import cfg
from anvil import colorizer
from anvil import component as comp
from anvil import date
from anvil import exceptions
from anvil import libvirt as lv
from anvil import log as logging
@@ -38,9 +37,9 @@ API_CONF = nhelper.API_CONF
DEF_VOL_PREFIX = nhelper.DEF_VOL_PREFIX
DEF_INSTANCE_PREFIX = nhelper.DEF_INSTANCE_PREFIX
DB_NAME = nhelper.DB_NAME
PASTE_CONF = nhelper.PASTE_CONF
# Normal conf
PASTE_CONF = 'nova-api-paste.ini'
PASTE_SOURCE_FN = 'api-paste.ini'
POLICY_CONF = 'policy.json'
LOGGING_SOURCE_FN = 'logging_sample.conf'
@@ -124,14 +123,11 @@ BIN_DIR = 'bin'
# This is a special conf
CLEANER_DATA_CONF = 'nova-clean.sh'
# Config keys we warm up so u won't be prompted later
WARMUP_PWS = [('rabbit', rhelper.PW_USER_PROMPT)]
class NovaMixin(object):
def known_options(self):
return set(['no-vnc', 'quantum', 'melange', 'no-db-sync'])
return set(['no-vnc', 'mq', 'quantum', 'melange', 'no-db-sync'])
def known_subsystems(self):
return list(SUBSYSTEMS)
@@ -151,21 +147,27 @@ class NovaMixin(object):
class NovaUninstaller(NovaMixin, comp.PythonUninstallComponent):
def __init__(self, *args, **kargs):
comp.PythonUninstallComponent.__init__(self, *args, **kargs)
self.bin_dir = sh.joinpths(self.app_dir, BIN_DIR)
self.virsh = lv.Virsh(self.cfg, self.distro)
def pre_uninstall(self):
self._clear_libvirt_domains()
self._clean_it()
def _filter_subsystems(self):
subs = set()
for name, values in self.subsystems.items():
if name in SUB_COMPONENT_NAME_MAP:
subs.add(name)
return subs
def _clean_it(self):
# These environment additions are important
# in that they eventually affect how this script runs
env = dict()
env['ENABLED_SERVICES'] = ",".join(self.desired_subsystems)
env['BIN_DIR'] = self.bin_dir
env['ENABLED_SERVICES'] = ",".join(self._filter_subsystems())
env['BIN_DIR'] = sh.joinpths(self.get_option('app_dir'), BIN_DIR)
env['VOLUME_NAME_PREFIX'] = self.cfg.getdefaulted('nova', 'volume_name_prefix', DEF_VOL_PREFIX)
cleaner_fn = sh.joinpths(self.bin_dir, CLEANER_DATA_CONF)
cleaner_fn = sh.joinpths(sh.joinpths(self.get_option('app_dir'), BIN_DIR), CLEANER_DATA_CONF)
if sh.isfile(cleaner_fn):
LOG.info("Cleaning up your system by running nova cleaner script: %s", colorizer.quote(cleaner_fn))
cmd = [cleaner_fn]
@@ -182,12 +184,8 @@ class NovaUninstaller(NovaMixin, comp.PythonUninstallComponent):
class NovaInstaller(NovaMixin, comp.PythonInstallComponent):
def __init__(self, *args, **kargs):
comp.PythonInstallComponent.__init__(self, *args, **kargs)
self.bin_dir = sh.joinpths(self.app_dir, BIN_DIR)
self.paste_conf_fn = self._get_target_config_name(PASTE_CONF)
self.volumes_enabled = False
self.volume_configurator = None
self.volumes_enabled = NVOL in self.desired_subsystems
self.xvnc_enabled = NXVNC in self.desired_subsystems
self.volumes_enabled = NVOL in self.subsystems
self.xvnc_enabled = NXVNC in self.subsystems
self.root_wrap_bin = sh.joinpths(self.distro.get_command_config('bin_dir'), 'nova-rootwrap')
self.volume_maker = None
if self.volumes_enabled:
@@ -196,7 +194,7 @@ class NovaInstaller(NovaMixin, comp.PythonInstallComponent):
def _get_symlinks(self):
links = comp.PythonInstallComponent._get_symlinks(self)
source_fn = sh.joinpths(self.cfg_dir, API_CONF)
source_fn = sh.joinpths(self.get_option('cfg_dir'), API_CONF)
links[source_fn] = sh.joinpths(self._get_link_dir(), API_CONF)
return links
@@ -207,10 +205,13 @@ class NovaInstaller(NovaMixin, comp.PythonInstallComponent):
self.volume_maker.verify()
def warm_configs(self):
warm_pws = list(WARMUP_PWS)
warm_pws = list()
mq_type = nhelper.canon_mq_type(self.get_option('mq'))
if mq_type == 'rabbit':
warm_pws.append(['rabbit', rhelper.PW_USER_PROMPT])
driver_canon = nhelper.canon_virt_driver(self.cfg.get('nova', 'virt_driver'))
if driver_canon == 'xenserver':
warm_pws.append(('xenapi_connection', 'the Xen API connection'))
warm_pws.append(['xenapi_connection', 'the Xen API connection'])
for pw_key, pw_prompt in warm_pws:
self.cfg.get_password(pw_key, pw_prompt)
@@ -232,9 +233,9 @@ class NovaInstaller(NovaMixin, comp.PythonInstallComponent):
def _setup_cleaner(self):
LOG.info("Configuring cleaner template: %s", colorizer.quote(CLEANER_DATA_CONF))
(_, contents) = utils.load_template(self.component_name, CLEANER_DATA_CONF)
(noop_fn, contents) = utils.load_template(self.name, CLEANER_DATA_CONF)
# FIXME, stop placing in checkout dir...
tgt_fn = sh.joinpths(self.bin_dir, CLEANER_DATA_CONF)
tgt_fn = sh.joinpths(sh.joinpths(self.get_option('app_dir'), BIN_DIR), CLEANER_DATA_CONF)
sh.write_file(tgt_fn, contents)
sh.chmod(tgt_fn, 0755)
self.tracewriter.file_touched(tgt_fn)
@@ -246,7 +247,7 @@ class NovaInstaller(NovaMixin, comp.PythonInstallComponent):
def _generate_nova_conf(self, root_wrapped):
conf_fn = self._get_target_config_name(API_CONF)
LOG.info("Generating dynamic content for nova: %s.", colorizer.quote(conf_fn))
nova_conf_contents = self.conf_maker.configure(root_wrapped)
nova_conf_contents = self.conf_maker.configure(fn=conf_fn, root_wrapped=root_wrapped)
self.tracewriter.dirs_made(*sh.mkdirslist(sh.dirname(conf_fn)))
self.tracewriter.cfg_file_written(sh.write_file(conf_fn, nova_conf_contents))
@@ -255,7 +256,7 @@ class NovaInstaller(NovaMixin, comp.PythonInstallComponent):
config_fn = PASTE_SOURCE_FN
elif config_fn == LOGGING_CONF:
config_fn = LOGGING_SOURCE_FN
fn = sh.joinpths(self.app_dir, 'etc', "nova", config_fn)
fn = sh.joinpths(self.get_option('app_dir'), 'etc', "nova", config_fn)
return (fn, sh.load_file(fn))
def _config_adjust_paste(self, contents, fn):
@@ -263,6 +264,7 @@ class NovaInstaller(NovaMixin, comp.PythonInstallComponent):
with io.BytesIO(contents) as stream:
config = cfg.RewritableConfigParser()
config.readfp(stream)
config.set('filter:authtoken', 'auth_host', params['endpoints']['admin']['host'])
config.set('filter:authtoken', 'auth_port', params['endpoints']['admin']['port'])
config.set('filter:authtoken', 'auth_protocol', params['endpoints']['admin']['protocol'])
@@ -274,6 +276,7 @@ class NovaInstaller(NovaMixin, comp.PythonInstallComponent):
config.set('filter:authtoken', 'admin_tenant_name', params['service_tenant'])
config.set('filter:authtoken', 'admin_user', params['service_user'])
config.set('filter:authtoken', 'admin_password', params['service_password'])
contents = config.stringify(fn)
return contents
@@ -303,8 +306,8 @@ class NovaInstaller(NovaMixin, comp.PythonInstallComponent):
def _get_param_map(self, config_fn):
mp = comp.PythonInstallComponent._get_param_map(self, config_fn)
mp['CFG_FILE'] = sh.joinpths(self.cfg_dir, API_CONF)
mp['BIN_DIR'] = self.bin_dir
mp['CFG_FILE'] = sh.joinpths(self.get_option('cfg_dir'), API_CONF)
mp['BIN_DIR'] = sh.joinpths(self.get_option('app_dir'), BIN_DIR)
return mp
def _generate_root_wrap(self):
@@ -335,20 +338,19 @@ class NovaInstaller(NovaMixin, comp.PythonInstallComponent):
class NovaRuntime(NovaMixin, comp.PythonRuntime):
def __init__(self, *args, **kargs):
comp.PythonRuntime.__init__(self, *args, **kargs)
self.bin_dir = sh.joinpths(self.app_dir, BIN_DIR)
self.wait_time = max(self.cfg.getint('DEFAULT', 'service_wait_seconds'), 1)
self.virsh = lv.Virsh(self.cfg, self.distro)
self.net_enabled = NNET in self.desired_subsystems
self.net_enabled = NNET in self.subsystems
def _do_network_init(self):
ran_fn = sh.joinpths(self.trace_dir, NET_INITED_FN)
ran_fn = sh.joinpths(self.get_option('trace_dir'), NET_INITED_FN)
if not sh.isfile(ran_fn) and self.net_enabled:
LOG.info("Creating your nova network to be used with instances.")
# Figure out the commands to run
mp = {}
cmds = []
mp['CFG_FILE'] = sh.joinpths(self.cfg_dir, API_CONF)
mp['BIN_DIR'] = self.bin_dir
mp['CFG_FILE'] = sh.joinpths(self.get_option('cfg_dir'), API_CONF)
mp['BIN_DIR'] = sh.joinpths(self.get_option('app_dir'), BIN_DIR)
if self.cfg.getboolean('nova', 'enable_fixed'):
# Create a fixed network
mp['FIXED_NETWORK_SIZE'] = self.cfg.getdefaulted('nova', 'fixed_network_size', '256')
@@ -381,11 +383,13 @@ class NovaRuntime(NovaMixin, comp.PythonRuntime):
def _get_apps_to_start(self):
apps = list()
for subsys in self.desired_subsystems:
apps.append({
'name': SUB_COMPONENT_NAME_MAP[subsys],
'path': sh.joinpths(self.bin_dir, SUB_COMPONENT_NAME_MAP[subsys]),
})
for name, values in self.subsystems.items():
if name in SUB_COMPONENT_NAME_MAP:
subsys = name
apps.append({
'name': SUB_COMPONENT_NAME_MAP[subsys],
'path': sh.joinpths(sh.joinpths(self.get_option('app_dir'), BIN_DIR), SUB_COMPONENT_NAME_MAP[subsys]),
})
return apps
def pre_start(self):
@@ -409,7 +413,7 @@ class NovaRuntime(NovaMixin, comp.PythonRuntime):
def _get_param_map(self, app_name):
params = comp.PythonRuntime._get_param_map(self, app_name)
params['CFG_FILE'] = sh.joinpths(self.cfg_dir, API_CONF)
params['CFG_FILE'] = sh.joinpths(self.get_option('cfg_dir'), API_CONF)
return params
def _get_app_options(self, app):

4
anvil/components/novnc.py
View File

@@ -68,7 +68,7 @@ class NoVNCRuntime(NoVNCMixin, comp.ProgramRuntime):
for app_name in APP_OPTIONS.keys():
apps.append({
'name': app_name,
'path': sh.joinpths(self.app_dir, UTIL_DIR, app_name),
'path': sh.joinpths(self.get_option('app_dir'), UTIL_DIR, app_name),
})
return apps
@@ -79,7 +79,7 @@ class NoVNCRuntime(NoVNCMixin, comp.ProgramRuntime):
if nova_name in self.instances:
# FIXME: Have to reach into the nova conf (puke)
nova_runtime = self.instances[nova_name]
root_params['NOVA_CONF'] = sh.joinpths(nova_runtime.cfg_dir, nova.API_CONF)
root_params['NOVA_CONF'] = sh.joinpths(nova_runtime.get_option('cfg_dir'), nova.API_CONF)
return root_params
def _get_app_options(self, app):

20
anvil/components/quantum.py
View File

@@ -144,14 +144,14 @@ class QuantumInstaller(QuantumMixin, comp.PkgInstallComponent):
def _get_source_config(self, config_fn):
if config_fn == PLUGIN_CONF:
srcfn = sh.joinpths(self.app_dir, 'etc', config_fn)
contents = sh.load_file(srcfn)
return (srcfn, contents)
src_fn = sh.joinpths(self.get_option('app_dir'), 'etc', config_fn)
contents = sh.load_file(src_fn)
return (src_fn, contents)
elif config_fn == AGENT_CONF:
# WHY U SO BURIED....
srcfn = sh.joinpths(self.app_dir, 'etc', 'quantum', 'plugins', 'openvswitch', config_fn)
contents = sh.load_file(srcfn)
return (srcfn, contents)
src_fn = sh.joinpths(self.get_option('app_dir'), 'etc', 'quantum', 'plugins', 'openvswitch', config_fn)
contents = sh.load_file(src_fn)
return (src_fn, contents)
else:
return comp.PkgInstallComponent._get_source_config(self, config_fn)
@@ -172,13 +172,13 @@ class QuantumRuntime(QuantumMixin, comp.ProgramRuntime):
if self.q_vswitch_service:
app_list.append({
'name': APP_Q_SERVER,
'path': sh.joinpths(self.app_dir, 'bin', APP_Q_SERVER),
'path': sh.joinpths(self.get_option('app_dir'), 'bin', APP_Q_SERVER),
})
if self.q_vswitch_agent:
app_list.append({
'name': APP_Q_AGENT,
# WHY U SO BURIED....
'path': sh.joinpths(self.app_dir, "quantum", "plugins", "openvswitch", 'agent', APP_Q_AGENT)
'path': sh.joinpths(self.get_option('app_dir'), "quantum", "plugins", "openvswitch", 'agent', APP_Q_AGENT)
})
return app_list
@@ -188,7 +188,7 @@ class QuantumRuntime(QuantumMixin, comp.ProgramRuntime):
def _get_param_map(self, app_name):
param_dict = comp.ProgramRuntime._get_param_map(self, app_name)
if app_name == APP_Q_AGENT:
param_dict['OVS_CONFIG_FILE'] = sh.joinpths(self.cfg_dir, AGENT_CONF)
param_dict['OVS_CONFIG_FILE'] = sh.joinpths(self.get_option('cfg_dir'), AGENT_CONF)
elif app_name == APP_Q_SERVER:
param_dict['QUANTUM_CONFIG_FILE'] = sh.joinpths(self.cfg_dir, QUANTUM_CONF)
param_dict['QUANTUM_CONFIG_FILE'] = sh.joinpths(self.get_option('cfg_dir'), QUANTUM_CONF)
return param_dict

37
anvil/components/rabbit.py
View File

@@ -18,6 +18,8 @@ from tempfile import TemporaryFile
from anvil import colorizer
from anvil import component as comp
from anvil import constants
from anvil import importer
from anvil import log as logging
from anvil import shell as sh
from anvil import utils
@@ -39,11 +41,11 @@ PW_USER_PROMPT = rhelper.PW_USER_PROMPT
class RabbitUninstaller(comp.PkgUninstallComponent):
def __init__(self, *args, **kargs):
comp.PkgUninstallComponent.__init__(self, *args, **kargs)
self.runtime = RabbitRuntime(*args, **kargs)
(runtime_cls, _) = self.distro.extract_component(self.component_name, 'running')
if not runtime_cls:
runtime_cls_name = self.siblings.get('running')
if not runtime_cls_name:
self.runtime = RabbitRuntime(*args, **kargs)
else:
runtime_cls = importer.import_entry_point(runtime_cls_name)
self.runtime = runtime_cls(*args, **kargs)
def pre_uninstall(self):
@@ -60,10 +62,11 @@ class RabbitUninstaller(comp.PkgUninstallComponent):
class RabbitInstaller(comp.PkgInstallComponent):
def __init__(self, *args, **kargs):
comp.PkgInstallComponent.__init__(self, *args, **kargs)
(runtime_cls, _) = self.distro.extract_component(self.component_name, 'running')
if not runtime_cls:
runtime_cls_name = self.siblings.get('running')
if not runtime_cls_name:
self.runtime = RabbitRuntime(*args, **kargs)
else:
runtime_cls = importer.import_entry_point(runtime_cls_name)
self.runtime = runtime_cls(*args, **kargs)
def warm_configs(self):
@@ -91,33 +94,27 @@ class RabbitRuntime(comp.EmptyRuntime):
self.redir_out = utils.make_bool(self.distro.get_command_config('rabbit-mq', 'redirect-outs'))
def start(self):
if self.status() != comp.STATUS_STARTED:
if self._status() != constants.STATUS_STARTED:
self._run_cmd(self.distro.get_command('rabbit-mq', 'start'))
return 1
else:
return 0
def status(self):
def _status(self):
# This has got to be the worst status output.
#
# I have ever seen (its like a weird mix json+crap)
run_result = sh.execute(
*self.distro.get_command('rabbit-mq', 'status'),
check_exit_code=False,
run_as_root=True)
if not run_result:
return comp.STATUS_UNKNOWN
(sysout, stderr) = run_result
combined = str(sysout) + str(stderr)
combined = combined.lower()
status_cmd = self.distro.get_command('rabbit-mq', 'status')
(sysout, stderr) = sh.execute(*status_cmd, check_exit_code=False, run_as_root=True)
combined = (str(sysout) + str(stderr)).lower()
if combined.find('nodedown') != -1 or \
combined.find("unable to connect to node") != -1 or \
combined.find('unrecognized') != -1:
return comp.STATUS_STOPPED
return constants.STATUS_STOPPED
elif combined.find('running_applications') != -1:
return comp.STATUS_STARTED
return constants.STATUS_STARTED
else:
return comp.STATUS_UNKNOWN
return constants.STATUS_UNKNOWN
def _run_cmd(self, cmd, check_exit=True):
# This seems to fix one of the bugs with rabbit mq starting and stopping
@@ -144,7 +141,7 @@ class RabbitRuntime(comp.EmptyRuntime):
return 1
def stop(self):
if self.status() != comp.STATUS_STOPPED:
if self._status() != constants.STATUS_STOPPED:
self._run_cmd(self.distro.get_command('rabbit-mq', 'stop'))
return 1
else:

91
anvil/components/swift.py
View File

@@ -70,12 +70,10 @@ WARMUP_PWS = [('service_token', 'the service admin token'),
class SwiftUninstaller(comp.PythonUninstallComponent):
def __init__(self, *args, **kargs):
comp.PythonUninstallComponent.__init__(self, *args, **kargs)
self.datadir = sh.joinpths(self.app_dir, self.cfg.getdefaulted('swift', 'data_location', 'data'))
self.bin_dir = sh.joinpths(self.app_dir, BIN_DIR)
self.logdir = sh.joinpths(self.datadir, LOG_DIR)
def pre_uninstall(self):
sh.umount(sh.joinpths(self.datadir, DEVICE_PATH))
data_dir = sh.joinpths(self.get_option('app_dir'), self.cfg.getdefaulted('swift', 'data_location', 'data'))
sh.umount(sh.joinpths(data_dir, DEVICE_PATH))
sh.replace_in(RSYNC_CONF_LOC, RSYNC_ON_OFF_RE, 'RSYNC_ENABLE=false', True)
def post_uninstall(self):
@@ -86,14 +84,6 @@ class SwiftUninstaller(comp.PythonUninstallComponent):
class SwiftInstaller(comp.PythonInstallComponent):
def __init__(self, *args, **kargs):
comp.PythonInstallComponent.__init__(self, *args, **kargs)
self.bin_dir = sh.joinpths(self.app_dir, BIN_DIR)
self.datadir = sh.joinpths(self.app_dir, self.cfg.getdefaulted('swift', 'data_location', 'data'))
self.logdir = sh.joinpths(self.datadir, LOG_DIR)
self.startmain_file = sh.joinpths(self.bin_dir, SWIFT_STARTMAIN)
self.makerings_file = sh.joinpths(self.bin_dir, SWIFT_MAKERINGS)
self.fs_dev = sh.joinpths(self.datadir, DEVICE_PATH)
self.fs_image = sh.joinpths(self.datadir, SWIFT_IMG)
self.auth_server = AUTH_SERVICE
def _get_download_locations(self):
places = list()
@@ -111,15 +101,18 @@ class SwiftInstaller(comp.PythonInstallComponent):
self.cfg.get_password(pw_key, prompt)
def _get_param_map(self, config_fn):
data_dir = sh.joinpths(self.get_option('app_dir'), self.cfg.getdefaulted('swift', 'data_location', 'data'))
cfg_dir = self.get_option('cfg_dir')
log_dir = sh.joinpths(data_dir, LOG_DIR)
return {
'USER': self.cfg.getdefaulted('swift', 'swift_user', sh.getuser()),
'GROUP': self.cfg.getdefaulted('swift', 'swift_group', sh.getgroupname()),
'SWIFT_DATA_LOCATION': self.datadir,
'SWIFT_CONFIG_LOCATION': self.cfg_dir,
'SWIFT_DATA_LOCATION': data_dir,
'SWIFT_CONFIG_LOCATION': cfg_dir,
'SERVICE_TOKEN': self.cfg.get('passwords', 'service_token'),
'AUTH_SERVER': self.auth_server,
'AUTH_SERVER': AUTH_SERVICE,
'SWIFT_HASH': self.cfg.get('passwords', 'swift_hash'),
'SWIFT_LOGDIR': self.logdir,
'SWIFT_LOGDIR': log_dir,
'SWIFT_PARTITION_POWER_SIZE': self.cfg.getdefaulted('swift', 'partition_power_size', '9'),
# Note: leave these alone, will be adjusted later
'NODE_PATH': '%NODE_PATH%',
@@ -128,24 +121,28 @@ class SwiftInstaller(comp.PythonInstallComponent):
}
def _create_data_location(self):
data_dir = sh.joinpths(self.get_option('app_dir'), self.cfg.getdefaulted('swift', 'data_location', 'data'))
fs_image = sh.joinpths(data_dir, SWIFT_IMG)
fs_dev = sh.joinpths(data_dir, DEVICE_PATH)
loop_size = self.cfg.get('swift', 'loopback_disk_size')
if not loop_size:
loop_size = DEF_LOOP_SIZE
else:
loop_size = utils.to_bytes(loop_size)
sh.create_loopback_file(fname=self.fs_image,
sh.create_loopback_file(fname=fs_image,
size=loop_size,
fs_type=FS_TYPE)
self.tracewriter.file_touched(self.fs_image)
sh.mount_loopback_file(self.fs_image, self.fs_dev, FS_TYPE)
sh.chown_r(self.fs_dev, sh.geteuid(), sh.getegid())
self.tracewriter.file_touched(fs_image)
sh.mount_loopback_file(fs_image, fs_dev, FS_TYPE)
sh.chown_r(fs_dev, sh.geteuid(), sh.getegid())
def _create_node_config(self, node_number, port):
data_dir = sh.joinpths(self.get_option('app_dir'), self.cfg.getdefaulted('swift', 'data_location', 'data'))
for t in ['object', 'container', 'account']:
src_fn = sh.joinpths(self.cfg_dir, '%s-server.conf' % t)
tgt_fn = sh.joinpths(self.cfg_dir, '%s-server/%d.conf' % (t, node_number))
src_fn = sh.joinpths(self.get_option('cfg_dir'), '%s-server.conf' % t)
tgt_fn = sh.joinpths(self.get_option('cfg_dir'), '%s-server/%d.conf' % (t, node_number))
adjustments = {
'%NODE_PATH%': sh.joinpths(self.datadir, str(node_number)),
'%NODE_PATH%': sh.joinpths(data_dir, str(node_number)),
'%BIND_PORT%': str(port),
'%LOG_FACILITY%': str(2 + node_number),
}
@@ -154,38 +151,46 @@ class SwiftInstaller(comp.PythonInstallComponent):
def _delete_templates(self):
for t in ['object', 'container', 'account']:
sh.unlink(sh.joinpths(self.cfg_dir, '%s-server.conf' % t))
sh.unlink(sh.joinpths(self.get_option('cfg_dir'), '%s-server.conf' % t))
def _create_nodes(self):
data_dir = sh.joinpths(self.get_option('app_dir'), self.cfg.getdefaulted('swift', 'data_location', 'data'))
fs_dev = sh.joinpths(data_dir, DEVICE_PATH)
for i in range(1, 5):
self.tracewriter.dirs_made(sh.mkdirslist(sh.joinpths(self.fs_dev, '%d/node' % i)))
link_tgt = sh.joinpths(self.datadir, str(i))
sh.symlink(sh.joinpths(self.fs_dev, str(i)), link_tgt)
self.tracewriter.dirs_made(sh.mkdirslist(sh.joinpths(fs_dev, '%d/node' % i)))
link_tgt = sh.joinpths(data_dir, str(i))
sh.symlink(sh.joinpths(fs_dev, str(i)), link_tgt)
self.tracewriter.symlink_made(link_tgt)
start_port = (6010 + (i - 1) * 5)
self._create_node_config(i, start_port)
self._delete_templates()
def _turn_on_rsync(self):
sh.symlink(sh.joinpths(self.cfg_dir, RSYNC_CONF), RSYNCD_CONF_LOC)
sh.symlink(sh.joinpths(self.get_option('cfg_dir'), RSYNC_CONF), RSYNCD_CONF_LOC)
self.tracewriter.symlink_made(RSYNCD_CONF_LOC)
sh.replace_in(RSYNC_CONF_LOC, RSYNC_ON_OFF_RE, 'RSYNC_ENABLE=true', True)
def _create_log_dirs(self):
self.tracewriter.dirs_made(*sh.mkdirslist(sh.joinpths(self.logdir, 'hourly')))
sh.symlink(sh.joinpths(self.cfg_dir, SYSLOG_CONF), SWIFT_RSYNC_LOC)
data_dir = sh.joinpths(self.get_option('app_dir'), self.cfg.getdefaulted('swift', 'data_location', 'data'))
cfg_dir = self.get_option('cfg_dir')
log_dir = sh.joinpths(data_dir, LOG_DIR)
self.tracewriter.dirs_made(*sh.mkdirslist(sh.joinpths(log_dir, 'hourly')))
sh.symlink(sh.joinpths(cfg_dir, SYSLOG_CONF), SWIFT_RSYNC_LOC)
self.tracewriter.symlink_made(SWIFT_RSYNC_LOC)
def _setup_binaries(self):
sh.move(sh.joinpths(self.cfg_dir, SWIFT_MAKERINGS), self.makerings_file)
sh.chmod(self.makerings_file, 0777)
self.tracewriter.file_touched(self.makerings_file)
sh.move(sh.joinpths(self.cfg_dir, SWIFT_STARTMAIN), self.startmain_file)
sh.chmod(self.startmain_file, 0777)
self.tracewriter.file_touched(self.startmain_file)
startmain_file = sh.joinpths(self.get_option('app_dir'), BIN_DIR, SWIFT_STARTMAIN)
makerings_file = sh.joinpths(self.get_option('app_dir'), BIN_DIR, SWIFT_MAKERINGS)
sh.move(sh.joinpths(self.get_option('cfg_dir'), SWIFT_MAKERINGS), makerings_file)
sh.chmod(makerings_file, 0777)
self.tracewriter.file_touched(makerings_file)
sh.move(sh.joinpths(self.get_option('cfg_dir'), SWIFT_STARTMAIN), startmain_file)
sh.chmod(startmain_file, 0777)
self.tracewriter.file_touched(startmain_file)
def _make_rings(self):
sh.execute(self.makerings_file, run_as_root=True)
makerings_file = sh.joinpths(self.get_option('app_dir'), BIN_DIR, SWIFT_MAKERINGS)
sh.execute(makerings_file, run_as_root=True)
def post_install(self):
self._create_data_location()
@@ -199,20 +204,20 @@ class SwiftInstaller(comp.PythonInstallComponent):
class SwiftRuntime(comp.PythonRuntime):
def __init__(self, *args, **kargs):
comp.PythonRuntime.__init__(self, *args, **kargs)
self.datadir = sh.joinpths(self.app_dir, self.cfg.getdefaulted('swift', 'data_location', 'data'))
self.bin_dir = sh.joinpths(self.app_dir, BIN_DIR)
self.logdir = sh.joinpths(self.datadir, LOG_DIR)
def start(self):
bin_dir = sh.joinpths(self.get_option('app_dir'), BIN_DIR)
sh.execute(*RSYSLOG_SERVICE_RESTART, run_as_root=True)
sh.execute(*RSYNC_SERVICE_RESTART, run_as_root=True)
swift_start_cmd = [sh.joinpths(self.bin_dir, SWIFT_INIT)] + ['all', 'start']
swift_start_cmd = [sh.joinpths(bin_dir, SWIFT_INIT)] + ['all', 'start']
sh.execute(*swift_start_cmd, run_as_root=True)
def stop(self):
swift_stop_cmd = [sh.joinpths(self.bin_dir, SWIFT_INIT)] + ['all', 'stop']
bin_dir = sh.joinpths(self.get_option('app_dir'), BIN_DIR)
swift_stop_cmd = [sh.joinpths(bin_dir, SWIFT_INIT)] + ['all', 'stop']
sh.execute(*swift_stop_cmd, run_as_root=True)
def restart(self):
swift_restart_cmd = [sh.joinpths(self.bin_dir, SWIFT_INIT)] + ['all', 'restart']
bin_dir = sh.joinpths(self.get_option('app_dir'), BIN_DIR)
swift_restart_cmd = [sh.joinpths(bin_dir, SWIFT_INIT)] + ['all', 'restart']
sh.execute(*swift_restart_cmd, run_as_root=True)

56
anvil/helpers/nova.py
View File

@@ -29,6 +29,9 @@ from anvil.helpers import db as dbhelper
LOG = logging.getLogger(__name__)
# Paste configuration
PASTE_CONF = 'nova-api-paste.ini'
# Special generated conf
API_CONF = 'nova.conf'
@@ -72,6 +75,9 @@ QUANTUM_OPENSWITCH_OPS = {
'quantum_use_dhcp': True,
}
# Known mq types
MQ_TYPES = ['rabbit', 'qpid', 'zeromq']
# Xenserver specific defaults
XS_DEF_INTERFACE = 'eth1'
XA_CONNECTION_ADDR = '169.254.0.1'
@@ -110,6 +116,11 @@ VG_LVREMOVE_CMD = [
'run_as_root': True}
]
def canon_mq_type(mq_type):
if not mq_type:
return ''
return str(mq_type).lower().strip()
def canon_virt_driver(virt_driver):
if not virt_driver:
@@ -170,9 +181,8 @@ def get_shared_params(cfgobj):
class VolumeConfigurator(object):
def __init__(self, installer):
self.installer = weakref.proxy(installer)
self.cfg = installer.cfg
self.app_dir = installer.app_dir
self.distro = installer.distro
self.cfg = self.installer.cfg
def setup_volumes(self):
self._setup_vol_groups()
@@ -183,7 +193,8 @@ class VolumeConfigurator(object):
def _setup_vol_groups(self):
LOG.info("Attempting to setup volume groups for nova volume management.")
mp = dict()
backing_file = self.cfg.getdefaulted('nova', 'volume_backing_file', sh.joinpths(self.app_dir, 'nova-volumes-backing-file'))
backing_file = self.cfg.getdefaulted('nova', 'volume_backing_file',
sh.joinpths(self.installer.get_option('app_dir'), 'nova-volumes-backing-file'))
vol_group = self.cfg.getdefaulted('nova', 'volume_group', 'nova-volumes')
backing_file_size = utils.to_bytes(self.cfg.getdefaulted('nova', 'volume_backing_file_size', '2052M'))
mp['VOLUME_GROUP'] = vol_group
@@ -249,15 +260,12 @@ class ConfConfigurator(object):
self.installer = weakref.proxy(installer)
self.cfg = installer.cfg
self.instances = installer.instances
self.component_dir = installer.component_dir
self.app_dir = installer.app_dir
self.tracewriter = installer.tracewriter
self.paste_conf_fn = installer.paste_conf_fn
self.paste_conf_fn = installer._get_target_config_name(PASTE_CONF)
self.distro = installer.distro
self.cfg_dir = installer.cfg_dir
self.options = installer.options
self.xvnc_enabled = installer.xvnc_enabled
self.volumes_enabled = installer.volumes_enabled
self.novnc_enabled = installer.get_option('no-vnc', False)
self.novnc_enabled = 'no-vnc' in installer.options
def _getbool(self, name):
@@ -289,11 +297,15 @@ class ConfConfigurator(object):
if flat_interface and not flat_interface in known_interfaces:
msg = "Libvirt flat interface %s is not a known interface (is it one of %s??)" % (flat_interface, ", ".join(known_interfaces))
raise exceptions.ConfigException(msg)
mq_type = canon_mq_type(self.installer.get_option('mq'))
if mq_type not in MQ_TYPES:
msg = "Unknown message queue type %s (is it one of %s??)" % (mq_type, ", ".join(MQ_TYPES))
raise exceptions.ConfigException(msg)
def configure(self, root_wrapped):
def configure(self, fn=API_CONF, root_wrapped=False):
# Everything built goes in here
nova_conf = Conf(API_CONF)
nova_conf = Conf(fn)
# Used more than once so we calculate it ahead of time
hostip = self.cfg.get('host', 'ip')
@@ -361,12 +373,20 @@ class ConfConfigurator(object):
nova_conf.add('ec2_dmz_host', self._getstr('ec2_dmz_host', hostip))
nova_conf.add('s3_host', hostip)
# How is your rabbit setup?
nova_conf.add('rabbit_host', self.cfg.getdefaulted('rabbit', 'rabbit_host', hostip))
nova_conf.add('rabbit_password', self.cfg.get("passwords", "rabbit"))
# How is your mq setup?
mq_type = canon_mq_type(self.installer.get_option('mq'))
if mq_type == 'rabbit':
nova_conf.add('rabbit_host', self.cfg.getdefaulted('rabbit', 'rabbit_host', hostip))
nova_conf.add('rabbit_password', self.cfg.get("passwords", "rabbit"))
nova_conf.add('rpc_backend', 'nova.rpc.impl_kombu')
elif mq_type == 'qpid':
nova_conf.add('rpc_backend', 'nova.rpc.impl_qpid')
elif mq_type == 'zeromq':
# TODO more needed???
nova_conf.add('rpc_backend', 'nova.rpc.impl_kombu')
# Where instances will be stored
instances_path = self._getstr('instances_path', sh.joinpths(self.component_dir, 'instances'))
instances_path = self._getstr('instances_path', sh.joinpths(self.installer.get_option('component_dir'), 'instances'))
self._configure_instances_path(instances_path, nova_conf)
# Is this a multihost setup?
@@ -471,7 +491,7 @@ class ConfConfigurator(object):
def _configure_network_settings(self, nova_conf):
# TODO this might not be right....
if 'quantum' in self.options:
if self.installer.get_option('quantum'):
nova_conf.add('network_manager', QUANTUM_MANAGER)
hostip = self.cfg.get('host', 'ip')
nova_conf.add('quantum_connection_host', self.cfg.getdefaulted('quantum', 'q_host', hostip))
@@ -479,7 +499,7 @@ class ConfConfigurator(object):
if self.cfg.get('quantum', 'q_plugin') == 'openvswitch':
for (key, value) in QUANTUM_OPENSWITCH_OPS.items():
nova_conf.add(key, value)
if 'melange' in self.options:
if self.installer.get_option('melange'):
nova_conf.add('quantum_ipam_lib', QUANTUM_IPAM_LIB)
nova_conf.add('use_melange_mac_generation', True)
nova_conf.add('melange_host', self.cfg.getdefaulted('melange', 'm_host', hostip))
@@ -489,7 +509,7 @@ class ConfConfigurator(object):
# Configs dhcp bridge stuff???
# TODO: why is this the same as the nova.conf?
nova_conf.add('dhcpbridge_flagfile', sh.joinpths(self.cfg_dir, API_CONF))
nova_conf.add('dhcpbridge_flagfile', sh.joinpths(self.installer.get_option('cfg_dir'), API_CONF))
# Network prefix for the IP network that all the projects for future VM guests reside on. Example: 192.168.0.0/12
nova_conf.add('fixed_range', self._getstr('fixed_range'))
@@ -574,8 +594,6 @@ class Conf(object):
self.name = name
def add(self, key, value, *values):
if not key:
raise exceptions.BadParamException("Can not add a empty/none/false key")
real_key = str(key)
real_value = ""
if len(values):

18
anvil/phase.py
View File

@@ -50,14 +50,24 @@ class PhaseRecorder(object):
phases = set()
if not sh.isfile(self.fn):
return phases
line_num = 1
for line in sh.load_file(self.fn).splitlines():
for i, line in enumerate(sh.load_file(self.fn).splitlines()):
line = line.strip()
if line:
data = json.loads(line)
if not isinstance(data, dict):
raise TypeError("Unknown phase entry in %s on line %s" % (self.fn, line_num))
raise TypeError("Unknown phase entry in %s on line %s" % (self.fn, i + 1))
if 'name' in data:
phases.add(data['name'])
line_num += 1
return phases
class NullPhaseRecorder(PhaseRecorder):
def __init__(self):
PhaseRecorder.__init__(None)
@contextmanager
def mark(self, phasename):
yield phasename
def list_phases(self):
return set()

9
anvil/pip.py
View File

@@ -38,13 +38,16 @@ class Packager(pack.Packager):
def _install(self, pip):
root_cmd = self._get_pip_command()
name_full = self._make_pip_name(pip['name'], pip.get('version'))
LOG.audit("Installing python package %r using pip command %s" % (name_full, root_cmd))
real_cmd = [root_cmd] + ['install'] + PIP_INSTALL_CMD_OPTS
options = pip.get('options')
if options:
if not isinstance(options, (list, tuple)):
options = [options]
LOG.debug("Using pip options: %s" % (options))
real_cmd += [str(options)]
real_cmd += [name_full]
for opt in options:
real_cmd.append("%s" % (opt))
LOG.audit("Installing python package %r using pip command %s" % (name_full, real_cmd))
real_cmd.append(name_full)
sh.execute(*real_cmd, run_as_root=True)
def _remove(self, pip):

4
anvil/runners/upstart.py
View File

@@ -16,10 +16,10 @@
import json
from anvil import constants
from anvil import date
from anvil import log as logging
from anvil import runner as base
from anvil import settings
from anvil import shell as sh
from anvil import trace as tr
from anvil import utils
@@ -83,7 +83,7 @@ class UpstartRunner(base.Runner):
params['COMPONENT_START_EVENT'] = self.component_name + START_EVENT_SUFFIX
params['COMPONENT_STOP_EVENT'] = self.component_name + STOP_EVENT_SUFFIX
params['PROGRAM_NAME'] = app_pth
params['AUTHOR'] = settings.PROG_NAME
params['AUTHOR'] = constants.PROG_NAME
if program_args:
escaped_args = list()
for opt in program_args:

9
anvil/settings.py
View File

@@ -17,12 +17,7 @@
import os
import sys
# What this program is called
PROG_NAME = 'anvil'
# Ip version constants for network ip detection
IPV4 = 'IPv4'
IPV6 = 'IPv6'
from anvil import constants
# RC files generated / used
RC_FN_TEMPL = "%s.rc"
@@ -33,7 +28,7 @@ CONFIG_DIR = os.path.join(BIN_DIR, "conf")
DISTRO_DIR = os.path.join(CONFIG_DIR, "distros")
TEMPLATE_DIR = os.path.join(CONFIG_DIR, "templates")
PERSONA_DIR = os.path.join(CONFIG_DIR, "personas")
CONFIG_NAME = 'anvil.ini'
CONFIG_NAME = constants.CONFIG_NAME
CONFIG_LOCATION = os.path.join(CONFIG_DIR, CONFIG_NAME)

13
anvil/utils.py
View File

@@ -31,6 +31,7 @@ import netifaces
import progressbar
import yaml
from anvil import constants
from anvil import colorizer
from anvil import date
from anvil import exceptions as excp
@@ -48,7 +49,7 @@ EXT_COMPONENT = re.compile(r"^\s*([\w-]+)(?:\((.*)\))?\s*$")
MONTY_PYTHON_TEXT_RE = re.compile("([a-z0-9A-Z\?!.,'\"]+)")
DEF_IP = "127.0.0.1"
IP_LOOKER = '8.8.8.8'
DEF_IP_VERSION = settings.IPV4
DEF_IP_VERSION = constants.IPV4
STAR_VERSION = 0
# Thx cowsay
@@ -326,11 +327,11 @@ def get_interfaces():
ip6 = interface_addresses.get(netifaces.AF_INET6)
if ip6:
# Just take the first
interface_info[settings.IPV6] = ip6[0]
interface_info[constants.IPV6] = ip6[0]
ip4 = interface_addresses.get(netifaces.AF_INET)
if ip4:
# Just take the first
interface_info[settings.IPV4] = ip4[0]
interface_info[constants.IPV4] = ip4[0]
# Note: there are others but this is good for now..
interfaces[intfc] = interface_info
return interfaces
@@ -388,8 +389,8 @@ def prettify_yaml(obj):
default_flow_style=False,
)
return formatted
def param_replace_deep(root, replacements, ignore_missing=False):
if isinstance(root, list):
new_list = []
@@ -726,7 +727,7 @@ def goodbye(worked):
print(msg)
def welcome(prog_name=settings.PROG_NAME.upper(), version_text=version.version_string()):
def welcome(prog_name=constants.PROG_NAME.upper(), version_text=version.version_string()):
lower = "| %s |" % (version_text)
welcome_header = _get_welcome_stack()
max_line_len = len(max(welcome_header.splitlines(), key=len))

2
anvil/version.py
View File

@@ -26,4 +26,4 @@ def version_string():
if FINAL:
return canonical_version_string()
else:
return '%s-dev' % (canonical_version_string(),)
return '%s-dev' % (canonical_version_string())

1
conf/anvil.ini
View File

@@ -93,7 +93,6 @@ glance_protocol = ${GLANCE_PROTOCOL:-http}
# Specify a comma-separated list of images to download and install into glance.
image_urls = http://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-uec.tar.gz,
http://smoser.brickies.net/ubuntu/ttylinux-uec/ttylinux-uec-amd64-11.2_2.6.35-15_1.tar.gz,
http://uec-images.ubuntu.com/oneiric/current/oneiric-server-cloudimg-amd64.tar.gz,
[nova]

4
conf/distros/fedora-16.yaml
View File

@@ -201,7 +201,7 @@ components:
version: 0.5*
pips:
- name: jsonschema
version: 0.2
version: "0.2"
glance-client:
action_classes:
install: anvil.components.glance_client:GlanceClientInstaller
@@ -279,7 +279,7 @@ components:
version: 0.5*
pips:
- name: django
version: 1.4
version: "1.4"
keystone:
action_classes:
install: anvil.components.keystone:KeystoneInstaller

24
conf/distros/rhel-6.yaml
View File

@@ -205,9 +205,9 @@ components:
version: 0.1.4
- name: pycrypto
options: --upgrade
version: '2.5'
version: '2.6'
- name: jsonschema
version: 0.2
version: "0.2"
glance-client:
action_classes:
install: anvil.components.glance_client:GlanceClientInstaller
@@ -263,7 +263,7 @@ components:
- name: SQLAlchemy
version: 0.7.5
- name: django
version: 1.4
version: "1.4"
- name: django-mailer
version: 0.1.0
- name: django-nose
@@ -276,7 +276,7 @@ components:
version: 1.1.2
- name: pycrypto
options: --upgrade
version: '2.5'
version: '2.6'
- name: python-cloudfiles
version: 1.7.9.3
- name: sqlalchemy-migrate
@@ -346,7 +346,7 @@ components:
version: '0.2'
- name: pycrypto
options: --upgrade
version: '2.5'
version: '2.6'
- name: sqlalchemy-migrate
version: 0.7.2
keystone-client:
@@ -463,6 +463,9 @@ components:
- name: sqlite
removable: false
version: 3.6*
- name: vconfig
removable: true
version: 1.9*
pips:
- name: Paste
version: 1.7.5.1
@@ -481,7 +484,7 @@ components:
version: 0.1.4
- name: pycrypto
options: --upgrade
version: '2.5'
version: '2.6'
- name: sqlalchemy-migrate
version: 0.7.2
subsystems:
@@ -562,6 +565,15 @@ components:
version: 1.2*
pips:
- name: cliff
qpid:
action_classes:
install: anvil.components.qpid:QpidInstaller
running: anvil.components.qpid:QpidRuntime
uninstall: anvil.components.qpid:QpidUninstaller
packages:
- name: qpid-cpp-server
removable: false
version: 0.14*
quantum:
action_classes:
install: anvil.components.quantum:QuantumInstaller

4
conf/distros/ubuntu-oneiric.yaml
View File

@@ -224,7 +224,7 @@ components:
- name: iso8601
version: 0.1.4
- name: jsonschema
version: 0.2
version: "0.2"
glance-client:
action_classes:
install: anvil.components.glance_client:GlanceClientInstaller
@@ -306,7 +306,7 @@ components:
- name: django-nose-selenium
version: 0.7.3
- name: pycrypto
version: 2.3
version: "2.6"
- name: python-cloudfiles
version: 1.7.9.3
keystone:

2
conf/distros/ubuntu-precise.yaml
View File

@@ -215,7 +215,7 @@ components:
version: 0.1*
pips:
- name: jsonschema
version: 0.2
version: "0.2"
glance-client:
action_classes:
install: anvil.components.glance_client:GlanceClientInstaller

8
conf/personas/devstack.sh.yaml
View File

@@ -10,10 +10,8 @@ components:
# Do this after glance is installed to override the old binary
- glance-client
- nova
- no-vnc
- quantum-client
- nova-client
- horizon
# Super client, so install after other clients
- openstack-client
description: Devstack.sh matching component installation.
@@ -22,8 +20,10 @@ options:
# This is the nova component name (we need this to hook into the nova conf...)
nova_component: nova
nova:
# We are enabling no-vnc (or trying to)
- no-vnc
# Signal nova we are using no-vnc
no-vnc: true
# Which message queue type should go here (rabbit or qpid)
mq: rabbit
subsystems:
glance:
- api