From ac292e8d547e69fc618583ab5debf8dd7fc71583 Mon Sep 17 00:00:00 2001 From: Anastasia Karpinska Date: Mon, 27 May 2013 10:28:56 +0400 Subject: [PATCH] Components __init__.py was splitted on actions base classes Change-Id: I6f4118b336a4c5edb1ef2bac51942118efe8c4a1 --- anvil/actions/install.py | 6 +- anvil/actions/status.py | 2 +- anvil/components/__init__.py | 1057 -------------------- anvil/{component.py => components/base.py} | 0 anvil/components/base_install.py | 626 ++++++++++++ anvil/components/base_runtime.py | 266 +++++ anvil/components/base_testing.py | 143 +++ anvil/components/cinder.py | 24 +- anvil/components/db.py | 32 +- anvil/components/glance.py | 33 +- anvil/components/glance_client.py | 19 +- anvil/components/horizon.py | 34 +- anvil/components/keystone.py | 27 +- anvil/components/keystone_client.py | 4 +- anvil/components/nova.py | 28 +- anvil/components/novnc.py | 12 +- anvil/components/openstack_client.py | 7 +- anvil/components/pkglist.py | 6 +- anvil/components/quantum.py | 14 +- anvil/components/rabbit.py | 30 +- anvil/components/swift_client.py | 5 +- anvil/packaging/rpm.py | 3 +- anvil/runners/__init__.py | 2 +- anvil/runners/fork.py | 2 +- conf/distros/rhel.yaml | 114 +-- 25 files changed, 1258 insertions(+), 1238 deletions(-) rename anvil/{component.py => components/base.py} (100%) create mode 100644 anvil/components/base_install.py create mode 100644 anvil/components/base_runtime.py create mode 100644 anvil/components/base_testing.py diff --git a/anvil/actions/install.py b/anvil/actions/install.py index a7beea16..272542a6 100644 --- a/anvil/actions/install.py +++ b/anvil/actions/install.py @@ -18,12 +18,12 @@ from StringIO import StringIO from anvil import action from anvil import colorizer -from anvil import components from anvil import log from anvil import pprint from anvil import shell as sh from anvil import utils +from anvil.components import base_install as binstall from anvil.action import PhaseFunctors LOG = log.getLogger(__name__) @@ -131,9 +131,9 @@ class InstallAction(action.Action): def capture_run(instance): instance_dependencies = {} - if isinstance(instance, (components.PkgInstallComponent)): + if isinstance(instance, (binstall.PkgInstallComponent)): instance_dependencies['packages'] = instance.packages - if isinstance(instance, (components.PythonInstallComponent)): + if isinstance(instance, (binstall.PythonInstallComponent)): instance_dependencies['pips'] = instance.pip_requires all_instance_dependencies[instance.name] = instance_dependencies diff --git a/anvil/actions/status.py b/anvil/actions/status.py index 06d4ef5e..ab2d249b 100644 --- a/anvil/actions/status.py +++ b/anvil/actions/status.py @@ -23,7 +23,7 @@ from anvil.action import PhaseFunctors LOG = log.getLogger(__name__) -from anvil.components import (STATUS_INSTALLED, STATUS_STARTED, +from anvil.components.base_runtime import (STATUS_INSTALLED, STATUS_STARTED, STATUS_STOPPED, STATUS_UNKNOWN) STATUS_COLOR_MAP = { diff --git a/anvil/components/__init__.py b/anvil/components/__init__.py index 74f74808..a7bfb005 100644 --- a/anvil/components/__init__.py +++ b/anvil/components/__init__.py @@ -13,1060 +13,3 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import functools -import os -import re -import weakref - -from anvil import cfg -from anvil import colorizer -from anvil import component -from anvil import decorators -from anvil import downloader as down -from anvil import exceptions as excp -from anvil import importer -from anvil import log as logging -from anvil import packager -from anvil import patcher -from anvil import shell as sh -from anvil import trace as tr -from anvil import utils - -from anvil.packaging import pip -from anvil.packaging import yum - -from anvil.packaging.helpers import pip_helper - -from anvil.components.configurators import base as conf - -LOG = logging.getLogger(__name__) - -#### -#### Utils... -#### - -# Cache of accessed packagers -_PACKAGERS = {} - -# Enviroment to run tests -DEFAULT_ENV = { - 'NOSE_WITH_OPENSTACK':'1', - 'NOSE_OPENSTACK_COLOR':'1', - 'NOSE_OPENSTACK_RED':'0.05', - 'NOSE_OPENSTACK_YELLOW':'0.025', - 'NOSE_OPENSTACK_SHOW_ELAPSED':'1', - 'NOSE_OPENSTACK_STDOUT':'1' } - -def make_packager(package, default_class, **kwargs): - packager_name = package.get('packager_name') or '' - packager_name = packager_name.strip() - if packager_name: - packager_cls = importer.import_entry_point(packager_name) - else: - packager_cls = default_class - if packager_cls in _PACKAGERS: - return _PACKAGERS[packager_cls] - p = packager_cls(**kwargs) - _PACKAGERS[packager_cls] = p - return p - - -# Remove any private keys from a package dictionary -def filter_package(pkg): - n_pkg = {} - for (k, v) in pkg.items(): - if not k or k.startswith("_"): - continue - else: - n_pkg[k] = v - return n_pkg - -#### -#### INSTALL CLASSES -#### - - -class PkgInstallComponent(component.Component): - def __init__(self, *args, **kargs): - component.Component.__init__(self, *args, **kargs) - trace_fn = tr.trace_filename(self.get_option('trace_dir'), 'created') - self.tracewriter = tr.TraceWriter(trace_fn, break_if_there=False) - self.configurator = conf.Configurator(self) - - def _get_download_config(self): - return None - - def _get_download_location(self): - key = self._get_download_config() - if not key: - return (None, None) - uri = self.get_option(key, default_value='').strip() - if not uri: - raise ValueError(("Could not find uri in config to download " - "from option %s") % (key)) - return (uri, self.get_option('app_dir')) - - def download(self): - (from_uri, target_dir) = self._get_download_location() - if not from_uri and not target_dir: - return [] - else: - uris = [from_uri] - utils.log_iterable(uris, logger=LOG, - header="Downloading from %s uris" % (len(uris))) - sh.mkdirslist(target_dir, tracewriter=self.tracewriter) - # This is used to delete what is downloaded (done before - # fetching to ensure its cleaned up even on download failures) - self.tracewriter.download_happened(target_dir, from_uri) - fetcher = down.GitDownloader(self.distro, from_uri, target_dir) - fetcher.download() - return uris - - def patch(self, section): - what_patches = self.get_option('patches', section) - (_from_uri, target_dir) = self._get_download_location() - if not what_patches: - what_patches = [] - canon_what_patches = [] - for path in what_patches: - if sh.isdir(path): - canon_what_patches.extend(sorted(sh.listdir(path, files_only=True))) - elif sh.isfile(path): - canon_what_patches.append(path) - if canon_what_patches: - patcher.apply_patches(canon_what_patches, target_dir) - - def config_params(self, config_fn): - mp = dict(self.params) - if config_fn: - mp['CONFIG_FN'] = config_fn - return mp - - @property - def packages(self): - pkg_list = self.get_option('packages', default_value=[]) - if not pkg_list: - pkg_list = [] - for name, values in self.subsystems.items(): - if 'packages' in values: - LOG.debug("Extending package list with packages for subsystem: %r", name) - pkg_list.extend(values.get('packages')) - return pkg_list - - def install(self): - LOG.debug('Preparing to install packages for: %r', self.name) - pkgs = self.packages - if pkgs: - pkg_names = set([p['name'] for p in pkgs]) - utils.log_iterable(pkg_names, logger=LOG, - header="Setting up %s distribution packages" % (len(pkg_names))) - with utils.progress_bar('Installing', len(pkgs)) as p_bar: - for (i, p) in enumerate(pkgs): - installer = make_packager(p, self.distro.package_manager_class, - distro=self.distro) - installer.install(p) - # Mark that this happened so that we can uninstall it - self.tracewriter.package_installed(filter_package(p)) - p_bar.update(i + 1) - - def pre_install(self): - pkgs = self.packages - for p in pkgs: - installer = make_packager(p, self.distro.package_manager_class, - distro=self.distro) - installer.pre_install(p, self.params) - - def post_install(self): - pkgs = self.packages - for p in pkgs: - installer = make_packager(p, self.distro.package_manager_class, - distro=self.distro) - installer.post_install(p, self.params) - - def _configure_files(self): - config_fns = self.configurator.config_files - if config_fns: - utils.log_iterable(config_fns, logger=LOG, - header="Configuring %s files" % (len(config_fns))) - for fn in config_fns: - tgt_fn = self.configurator.target_config(fn) - sh.mkdirslist(sh.dirname(tgt_fn), tracewriter=self.tracewriter) - (source_fn, contents) = self.configurator.source_config(fn) - LOG.debug("Configuring file %s ---> %s.", (source_fn), (tgt_fn)) - contents = self.configurator.config_param_replace(fn, contents, self.config_params(fn)) - contents = self.configurator.config_adjust(contents, fn) - sh.write_file(tgt_fn, contents, tracewriter=self.tracewriter) - return len(config_fns) - - def _configure_symlinks(self): - links = self.configurator.symlinks - if not links: - return 0 - # This sort happens so that we link in the correct order - # although it might not matter. Either way. We ensure that the right - # order happens. Ie /etc/blah link runs before /etc/blah/blah - link_srcs = sorted(links.keys()) - link_srcs.reverse() - link_nice = [] - for source in link_srcs: - links_to_be = links[source] - for link in links_to_be: - link_nice.append("%s => %s" % (link, source)) - utils.log_iterable(link_nice, logger=LOG, - header="Creating %s sym-links" % (len(link_nice))) - links_made = 0 - for source in link_srcs: - links_to_be = links[source] - for link in links_to_be: - try: - LOG.debug("Symlinking %s to %s.", link, source) - sh.symlink(source, link, tracewriter=self.tracewriter) - links_made += 1 - except (IOError, OSError) as e: - LOG.warn("Symlinking %s to %s failed: %s", colorizer.quote(link), colorizer.quote(source), e) - return links_made - - def configure(self): - return self._configure_files() + self._configure_symlinks() - - -class PythonInstallComponent(PkgInstallComponent): - def __init__(self, *args, **kargs): - PkgInstallComponent.__init__(self, *args, **kargs) - self.requires_files = [ - sh.joinpths(self.get_option('app_dir'), 'tools', 'pip-requires'), - ] - if self.get_bool_option('use_tests_requires', default_value=True): - self.requires_files.append(sh.joinpths(self.get_option('app_dir'), 'tools', 'test-requires')) - - def _get_download_config(self): - return 'get_from' - - @property - def python_directories(self): - py_dirs = {} - app_dir = self.get_option('app_dir') - if sh.isdir(app_dir): - py_dirs[self.name] = app_dir - return py_dirs - - @property - def packages(self): - pkg_list = super(PythonInstallComponent, self).packages - if not pkg_list: - pkg_list = [] - pkg_list.extend(self._get_mapped_packages()) - return pkg_list - - @property - def pips_to_packages(self): - pip_pkg_list = self.get_option('pip_to_package', default_value=[]) - if not pip_pkg_list: - pip_pkg_list = [] - return pip_pkg_list - - @property - def pip_requires(self): - all_pips = [] - for fn in self.requires_files: - all_pips.extend(self._extract_pip_requires(fn)) - return all_pips - - def _match_pip_requires(self, pip_req): - - def pip_use(who, there_pip): - if there_pip.key != pip_req.key: - return False - if not len(pip_req.specs): - # No version/restrictions specified - return True - there_version = None - if not there_pip.specs or there_pip == pip_req: - return True - # Different possibly incompat. versions found... - if there_version is None: - # Assume pip will install the correct version anyway - if who != self.name: - msg = ("Component %r asked for package '%s'" - " and '%s' is being selected from %r instead...") - LOG.debug(msg, self.name, pip_req, there_pip, who) - return True - else: - if who != self.name: - msg = ("Component %r provides package '%s'" - " but '%s' is being asked for by %r instead...") - LOG.warn(msg, who, there_pip, pip_req, self.name) - return False - - LOG.debug("Attempting to find who satisfies pip requirement '%s'", pip_req) - - # Try to find it in anyones pip -> pkg list - all_pip_2_pkgs = { - self.name: self.pips_to_packages, - } - # Gather them all (but only if they activate before me) - # since if they activate after, we can't depend on it - # to satisfy our requirement... - for (name, c) in self.instances.items(): - if c is self or not c.activated: - continue - if isinstance(c, (PythonInstallComponent)): - all_pip_2_pkgs[name] = c.pips_to_packages - for (who, pips_2_pkgs) in all_pip_2_pkgs.items(): - for pip_info in pips_2_pkgs: - there_pip = pip.extract_requirement(pip_info) - if not pip_use(who, there_pip): - continue - LOG.debug("Matched pip->pkg '%s' from component %r", there_pip, who) - return (dict(pip_info.get('package')), False) - - # Ok nobody had it in a pip->pkg mapping - # but see if they had it in there pip collection - all_pips = { - self.name: self._base_pips(), # Use base pips to avoid recursion... - } - for (name, c) in self.instances.items(): - if not c.activated or c is self: - continue - if isinstance(c, (PythonInstallComponent)): - all_pips[name] = c._base_pips() # pylint: disable=W0212 - for (who, there_pips) in all_pips.items(): - for pip_info in there_pips: - there_pip = pip.extract_requirement(pip_info) - if not pip_use(who, there_pip): - continue - LOG.debug("Matched pip '%s' from component %r", there_pip, who) - return (dict(pip_info), True) - - # Ok nobody had it in there pip->pkg mapping or pip mapping - # but now lets see if we can automatically find - # a pip->pkg mapping for them using the good ole' - # rpm/yum database. - installer = make_packager({}, self.distro.package_manager_class, - distro=self.distro) - - # TODO(harlowja): make this better - if installer and hasattr(installer, 'match_pip_2_package'): - try: - dist_pkg = installer.match_pip_2_package(pip_req) - if dist_pkg: - pkg_info = { - 'name': str(dist_pkg.name), - 'version': str(dist_pkg.version), - '__requirement': dist_pkg, - } - LOG.debug("Auto-matched (dist) %s -> %s", pip_req, dist_pkg) - return (pkg_info, False) - except excp.DependencyException as e: - LOG.warn("Unable to automatically map pip to package: %s", e) - - # Ok still nobody has it, search pypi... - pypi_pkg = pip_helper.find_pypi_match(pip_req) - if pypi_pkg: - pkg_info = { - 'name': str(pypi_pkg.key), - '__requirement': pypi_pkg, - } - try: - pkg_info['version'] = pypi_pkg.specs[0][1] - except IndexError: - pass - LOG.debug("Auto-matched (pypi) %s -> %s", pip_req, pypi_pkg) - return (pkg_info, True) - - return (None, False) - - def _get_mapped_packages(self): - add_on_pkgs = [] - all_pips = self.pip_requires - for details in all_pips: - pkg_info = details['package'] - from_pip = details['from_pip'] - if from_pip or not pkg_info: - continue - # Keep the initial requirement - pkg_info = dict(pkg_info) - pkg_info['__requirement'] = details['requirement'] - add_on_pkgs.append(pkg_info) - return add_on_pkgs - - def _get_mapped_pips(self): - add_on_pips = [] - all_pips = self.pip_requires - for details in all_pips: - pkg_info = details['package'] - from_pip = details['from_pip'] - if not from_pip or not pkg_info: - continue - # Keep the initial requirement - pkg_info = dict(pkg_info) - pkg_info['__requirement'] = details['requirement'] - add_on_pips.append(pkg_info) - return add_on_pips - - def _base_pips(self): - pip_list = self.get_option('pips', default_value=[]) - if not pip_list: - pip_list = [] - for (name, values) in self.subsystems.items(): - if 'pips' in values: - LOG.debug("Extending pip list with pips for subsystem: %r" % (name)) - pip_list.extend(values.get('pips')) - return pip_list - - @property - def pips(self): - pip_list = self._base_pips() - pip_list.extend(self._get_mapped_pips()) - return pip_list - - def _install_pips(self): - pips = self.pips - if pips: - pip_names = set([p['name'] for p in pips]) - utils.log_iterable(pip_names, logger=LOG, - header="Setting up %s python packages" % (len(pip_names))) - with utils.progress_bar('Installing', len(pips)) as p_bar: - for (i, p) in enumerate(pips): - installer = make_packager(p, pip.Packager, - distro=self.distro) - installer.install(p) - # Note that we did it so that we can remove it... - self.tracewriter.pip_installed(filter_package(p)) - p_bar.update(i + 1) - - def _clean_pip_requires(self): - # Fixup these files if they exist, sometimes they have 'junk' in them - # that anvil will install instead of pip or setup.py and we don't want - # the setup.py file to attempt to install said dependencies since it - # typically picks locations that either are not what we desire or if - # said file contains editables, it may even pick external source directories - # which is what anvil is setting up as well... - req_fns = [f for f in self.requires_files if sh.isfile(f)] - if req_fns: - utils.log_iterable(req_fns, logger=LOG, - header="Adjusting %s pip 'requires' files" % (len(req_fns))) - for fn in req_fns: - old_lines = sh.load_file(fn).splitlines() - new_lines = self._filter_pip_requires(fn, old_lines) - contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(), "\n".join(new_lines)) - sh.write_file_and_backup(fn, contents) - return len(req_fns) - - def _filter_pip_requires(self, fn, lines): - # The default does no filtering except to ensure that said lines are valid... - return lines - - def pre_install(self): - self._verify_pip_requires() - PkgInstallComponent.pre_install(self) - for p in self.pips: - installer = make_packager(p, pip.Packager, - distro=self.distro) - installer.pre_install(p, self.params) - - def post_install(self): - PkgInstallComponent.post_install(self) - for p in self.pips: - installer = make_packager(p, pip.Packager, - distro=self.distro) - installer.post_install(p, self.params) - - def _install_python_setups(self): - py_dirs = self.python_directories - if py_dirs: - real_dirs = {} - for (name, wkdir) in py_dirs.items(): - real_dirs[name] = wkdir - if not real_dirs[name]: - real_dirs[name] = self.get_option('app_dir') - utils.log_iterable(real_dirs.values(), logger=LOG, - header="Setting up %s python directories" % (len(real_dirs))) - setup_cmd = self.distro.get_command('python', 'setup') - for (name, working_dir) in real_dirs.items(): - sh.mkdirslist(working_dir, tracewriter=self.tracewriter) - setup_fn = sh.joinpths(self.get_option('trace_dir'), "%s.python.setup" % (name)) - sh.execute(*setup_cmd, cwd=working_dir, run_as_root=True, - stderr_fn='%s.stderr' % (setup_fn), - stdout_fn='%s.stdout' % (setup_fn), - tracewriter=self.tracewriter) - self.tracewriter.py_installed(name, working_dir) - - def _python_install(self): - self._install_pips() - self._install_python_setups() - - @decorators.memoized - def _extract_pip_requires(self, fn): - if not sh.isfile(fn): - return [] - LOG.debug("Resolving dependencies from %s.", colorizer.quote(fn)) - pips_needed = pip_helper.parse_requirements(sh.load_file(fn)) - matchings = [] - for req in pips_needed: - (pkg_info, from_pip) = self._match_pip_requires(req) - matchings.append({ - 'requirement': req, - 'package': pkg_info, - 'from_pip': from_pip, - 'needed_by': fn, - }) - return matchings - - def _verify_pip_requires(self): - all_pips = self.pip_requires - for details in all_pips: - req = details['requirement'] - needed_by = details['needed_by'] - pkg_info = details['package'] - if not pkg_info: - raise excp.DependencyException(("Pip dependency '%s' needed by '%s' is not translatable to a listed" - " (from this or previously activated components) pip package" - ' or a pip->package mapping!') % (req, needed_by)) - - def install(self): - PkgInstallComponent.install(self) - self._python_install() - - def configure(self): - configured_am = PkgInstallComponent.configure(self) - configured_am += self._clean_pip_requires() - return configured_am - - -#### -#### RUNTIME CLASSES -#### - -DEFAULT_RUNNER = 'anvil.runners.fork:ForkRunner' - -#### -#### STATUS CONSTANTS -#### -STATUS_INSTALLED = 'installed' -STATUS_STARTED = "started" -STATUS_STOPPED = "stopped" -STATUS_UNKNOWN = "unknown" - - -class ProgramStatus(object): - def __init__(self, status, name=None, details=''): - self.name = name - self.status = status - self.details = details - - -class Program(object): - def __init__(self, name, path=None, working_dir=None, argv=None): - self.name = name - if path is None: - self.path = name - else: - self.path = path - self.working_dir = working_dir - if argv is None: - self.argv = tuple() - else: - self.argv = tuple(argv) - - def __str__(self): - what = str(self.name) - if self.path: - what += " (%s)" % (self.path) - return what - - -class ProgramRuntime(component.Component): - @property - def applications(self): - # A list of applications since a single component sometimes - # has a list of programs to start (ie nova) instead of a single application (ie the db) - return [] - - def restart(self): - # How many applications restarted - return 0 - - def post_start(self): - pass - - def pre_start(self): - pass - - def statii(self): - # A list of statuses since a single component sometimes - # has a list of programs to report on (ie nova) instead of a single application (ie the db) - return [] - - def start(self): - # How many applications started - return 0 - - def stop(self): - # How many applications stopped - return 0 - - # TODO(harlowja): seems like this could be a mixin? - def wait_active(self, between_wait=1, max_attempts=5): - # Attempt to wait until all potentially started applications - # are actually started (for whatever defintion of started is applicable) - # for up to a given amount of attempts and wait time between attempts. - num_started = len(self.applications) - if not num_started: - raise excp.StatusException("No %r programs started, can not wait for them to become active..." % (self.name)) - - def waiter(try_num): - LOG.info("Waiting %s seconds for component %s programs to start.", between_wait, colorizer.quote(self.name)) - LOG.info("Please wait...") - sh.sleep(between_wait) - - for i in range(0, max_attempts): - statii = self.statii() - if len(statii) >= num_started: # >= if someone reports more than started... - not_worked = [] - for p in statii: - if p.status != STATUS_STARTED: - not_worked.append(p) - if len(not_worked) == 0: - return - else: - # Eck less applications were found with status then what were started! - LOG.warn("%s less applications reported status than were actually started!", - num_started - len(statii)) - waiter(i + 1) - - tot_time = max(0, (between_wait * max_attempts)) - raise excp.StatusException("Failed waiting %s seconds for component %r programs to become active..." - % (tot_time, self.name)) - - -class EmptyRuntime(ProgramRuntime): - pass - - -class PythonRuntime(ProgramRuntime): - def __init__(self, *args, **kargs): - ProgramRuntime.__init__(self, *args, **kargs) - start_trace = tr.trace_filename(self.get_option('trace_dir'), 'start') - self.tracewriter = tr.TraceWriter(start_trace, break_if_there=True) - self.tracereader = tr.TraceReader(start_trace) - - def app_params(self, program): - params = dict(self.params) - if program and program.name: - params['APP_NAME'] = str(program.name) - return params - - def start(self): - # Perform a check just to make sure said programs aren't already started and bail out - # so that it we don't unintentionally start new ones and thus causing confusion for all - # involved... - what_may_already_be_started = [] - try: - what_may_already_be_started = self.tracereader.apps_started() - except excp.NoTraceException: - pass - if what_may_already_be_started: - msg = "%s programs of component %s may already be running, did you forget to stop those?" - raise excp.StartException(msg % (len(what_may_already_be_started), self.name)) - - # Select how we are going to start it and get on with the show... - runner_entry_point = self.get_option("run_type", default_value=DEFAULT_RUNNER) - starter_args = [self, runner_entry_point] - starter = importer.construct_entry_point(runner_entry_point, *starter_args) - amount_started = 0 - for program in self.applications: - self._start_app(program, starter) - amount_started += 1 - return amount_started - - def _start_app(self, program, starter): - app_working_dir = program.working_dir - if not app_working_dir: - app_working_dir = self.get_option('app_dir') - - # Un-templatize whatever argv (program options) the program has specified - # with whatever program params were retrieved to create the 'real' set - # of program options (if applicable) - app_params = self.app_params(program) - if app_params: - app_argv = [utils.expand_template(arg, app_params) for arg in program.argv] - else: - app_argv = program.argv - LOG.debug("Starting %r using a %r", program.name, starter) - - # TODO(harlowja): clean this function params up (should just take a program) - details_path = starter.start(program.name, - app_pth=program.path, - app_dir=app_working_dir, - opts=app_argv) - - # This trace is used to locate details about what/how to stop - LOG.info("Started program %s under component %s.", colorizer.quote(program.name), self.name) - self.tracewriter.app_started(program.name, details_path, starter.name) - - def _locate_investigators(self, applications_started): - # Recreate the runners that can be used to dive deeper into the applications list - # that was started (a 3 tuple of (name, trace, who_started)). - investigators_created = {} - to_investigate = [] - for (name, _trace, who_started) in applications_started: - investigator = investigators_created.get(who_started) - if investigator is None: - try: - investigator_args = [self, who_started] - investigator = importer.construct_entry_point(who_started, *investigator_args) - investigators_created[who_started] = investigator - except RuntimeError as e: - LOG.warn("Could not load class %s which should be used to investigate %s: %s", - colorizer.quote(who_started), colorizer.quote(name), e) - continue - to_investigate.append((name, investigator)) - return to_investigate - - def stop(self): - # Anything to stop in the first place?? - what_was_started = [] - try: - what_was_started = self.tracereader.apps_started() - except excp.NoTraceException: - pass - if not what_was_started: - return 0 - - # Get the investigators/runners which can be used - # to actually do the stopping and attempt to perform said stop. - applications_stopped = [] - for (name, handler) in self._locate_investigators(what_was_started): - handler.stop(name) - applications_stopped.append(name) - if applications_stopped: - utils.log_iterable(applications_stopped, - header="Stopped %s programs started under %s component" % (len(applications_stopped), self.name), - logger=LOG) - - # Only if we stopped the amount which was supposedly started can - # we actually remove the trace where those applications have been - # marked as started in (ie the connection back to how they were started) - if len(applications_stopped) < len(what_was_started): - diff = len(what_was_started) - len(applications_stopped) - LOG.warn(("%s less applications were stopped than were started, please check out %s" - " to stop these program manually."), diff, colorizer.quote(self.tracereader.filename(), quote_color='yellow')) - else: - sh.unlink(self.tracereader.filename()) - - return len(applications_stopped) - - def statii(self): - # Anything to get status on in the first place?? - what_was_started = [] - try: - what_was_started = self.tracereader.apps_started() - except excp.NoTraceException: - pass - if not what_was_started: - return [] - - # Get the investigators/runners which can be used - # to actually do the status inquiry and attempt to perform said inquiry. - statii = [] - for (name, handler) in self._locate_investigators(what_was_started): - (status, details) = handler.status(name) - statii.append(ProgramStatus(name=name, - status=status, - details=details)) - return statii - - -#### -#### UNINSTALL CLASSES -#### - -class PkgUninstallComponent(component.Component): - def __init__(self, *args, **kargs): - component.Component.__init__(self, *args, **kargs) - trace_fn = tr.trace_filename(self.get_option('trace_dir'), 'created') - self.tracereader = tr.TraceReader(trace_fn) - self.purge_packages = kargs.get('purge_packages') - - def unconfigure(self): - self._unconfigure_links() - - def _unconfigure_links(self): - sym_files = self.tracereader.symlinks_made() - if sym_files: - utils.log_iterable(sym_files, logger=LOG, - header="Removing %s symlink files" % (len(sym_files))) - for fn in sym_files: - sh.unlink(fn, run_as_root=True) - - def uninstall(self): - self._uninstall_pkgs() - self._uninstall_files() - - def post_uninstall(self): - self._uninstall_dirs() - - def pre_uninstall(self): - pass - - def _uninstall_pkgs(self): - pkgs = self.tracereader.packages_installed() - if pkgs: - pkg_names = set([p['name'] for p in pkgs]) - utils.log_iterable(pkg_names, logger=LOG, - header="Potentially removing %s distribution packages" % (len(pkg_names))) - which_removed = [] - with utils.progress_bar('Uninstalling', len(pkgs), reverse=True) as p_bar: - for (i, p) in enumerate(pkgs): - uninstaller = make_packager(p, self.distro.package_manager_class, - distro=self.distro, - remove_default=self.purge_packages) - if uninstaller.remove(p): - which_removed.append(p['name']) - p_bar.update(i + 1) - utils.log_iterable(which_removed, logger=LOG, - header="Actually removed %s distribution packages" % (len(which_removed))) - - def _uninstall_files(self): - files_touched = self.tracereader.files_touched() - if files_touched: - utils.log_iterable(files_touched, logger=LOG, - header="Removing %s miscellaneous files" % (len(files_touched))) - for fn in files_touched: - sh.unlink(fn, run_as_root=True) - - def _uninstall_dirs(self): - dirs_made = self.tracereader.dirs_made() - dirs_alive = filter(sh.isdir, dirs_made) - if dirs_alive: - utils.log_iterable(dirs_alive, logger=LOG, - header="Removing %s created directories" % (len(dirs_alive))) - for dir_name in dirs_alive: - sh.deldir(dir_name, run_as_root=True) - - -class PythonUninstallComponent(PkgUninstallComponent): - - def uninstall(self): - self._uninstall_python() - self._uninstall_pips() - PkgUninstallComponent.uninstall(self) - - def _uninstall_pips(self): - pips = self.tracereader.pips_installed() - if pips: - pip_names = set([p['name'] for p in pips]) - utils.log_iterable(pip_names, logger=LOG, - header="Potentially removing %s python packages" % (len(pip_names))) - which_removed = [] - with utils.progress_bar('Uninstalling', len(pips), reverse=True) as p_bar: - for (i, p) in enumerate(pips): - try: - uninstaller = make_packager(p, pip.Packager, - distro=self.distro, - remove_default=self.purge_packages) - if uninstaller.remove(p): - which_removed.append(p['name']) - except excp.ProcessExecutionError as e: - # NOTE(harlowja): pip seems to die if a pkg isn't there even in quiet mode - combined = (str(e.stderr) + str(e.stdout)) - if not re.search(r"not\s+installed", combined, re.I): - raise - p_bar.update(i + 1) - utils.log_iterable(which_removed, logger=LOG, - header="Actually removed %s python packages" % (len(which_removed))) - - def _uninstall_python(self): - py_listing = self.tracereader.py_listing() - if py_listing: - py_listing_dirs = set() - for (_name, where) in py_listing: - py_listing_dirs.add(where) - utils.log_iterable(py_listing_dirs, logger=LOG, - header="Uninstalling %s python setups" % (len(py_listing_dirs))) - unsetup_cmd = self.distro.get_command('python', 'unsetup') - for where in py_listing_dirs: - if sh.isdir(where): - sh.execute(*unsetup_cmd, cwd=where, run_as_root=True) - else: - LOG.warn("No python directory found at %s - skipping", colorizer.quote(where, quote_color='red')) - - -#### -#### TESTING CLASSES -#### - - -class EmptyTestingComponent(component.Component): - def run_tests(self): - return - - def show_coverage(self): - return - -class PythonTestingComponent(component.Component): - def __init__(self, *args, **kargs): - component.Component.__init__(self, *args, **kargs) - self.helper = pip_helper.Helper(self.distro) - - def _get_test_exclusions(self): - return self.get_option('exclude_tests', default_value=[]) - - def _get_test_dir_exclusions(self): - return self.get_option('exclude_tests_dir', default_value=[]) - - def _use_run_tests(self): - return True - - def _get_test_command(self): - # See: http://docs.openstack.org/developer/nova/devref/unit_tests.html - # And: http://wiki.openstack.org/ProjectTestingInterface - app_dir = self.get_option('app_dir') - - cmd = ['coverage', 'run', '/usr/bin/nosetests'] - # See: $ man nosetests - if not colorizer.color_enabled(): - cmd.append('--openstack-nocolor') - else: - cmd.append('--openstack-color') - if self.get_bool_option("verbose", default_value=True): - cmd.append('--verbosity=2') - cmd.append('--detailed-errors') - else: - cmd.append('--verbosity=1') - for e in self._get_test_exclusions(): - cmd.append('--exclude=%s' % (e)) - for e in self._get_test_dir_exclusions(): - cmd.append('--exclude-dir=%s' % (e)) - xunit_fn = self.get_option("xunit_filename") - if xunit_fn: - cmd.append("--with-xunit") - cmd.append("--xunit-file=%s" % (xunit_fn)) - LOG.debug("Running tests: %s" % cmd) - return cmd - - def _use_pep8(self): - # Seems like the varying versions are borking pep8 from working... - i_sibling = self.siblings.get('install') - # Check if whats installed actually matches - pep8_wanted = None - if isinstance(i_sibling, (PythonInstallComponent)): - for p in i_sibling.pip_requires: - req = p['requirement'] - if req.key == "pep8": - pep8_wanted = req - break - if not pep8_wanted: - # Doesn't matter since its not wanted anyway - return True - pep8_there = self.helper.get_installed('pep8') - if not pep8_there: - # Hard to use it if it isn't there... - LOG.warn("Pep8 version mismatch, none is installed but %s is wanting %s", - self.name, pep8_wanted) - return False - if not (pep8_there == pep8_wanted): - # Versions not matching, this is causes pep8 to puke when it doesn't need to - # so skip it from running in the first place... - LOG.warn("Pep8 version mismatch, installed is %s but %s is applying %s", - pep8_there, self.name, pep8_wanted) - return False - return self.get_bool_option('use_pep8', default_value=True) - - def _get_env(self): - env_addons = DEFAULT_ENV.copy() - tox_fn = sh.joinpths(self.get_option('app_dir'), 'tox.ini') - if sh.isfile(tox_fn): - # Suck out some settings from the tox file - try: - tox_cfg = cfg.BuiltinConfigParser(fns=[tox_fn]) - env_values = tox_cfg.get('testenv', 'setenv') or '' - for env_line in env_values.splitlines(): - env_line = env_line.strip() - env_line = env_line.split("#")[0].strip() - if not env_line: - continue - env_entry = env_line.split('=', 1) - if len(env_entry) == 2: - (name, value) = env_entry - name = name.strip() - value = value.strip() - if name.lower() != 'virtual_env': - env_addons[name] = value - if env_addons: - LOG.debug("From %s we read in %s environment settings:", tox_fn, len(env_addons)) - utils.log_object(env_addons, logger=LOG, level=logging.DEBUG) - except IOError: - pass - - if not colorizer.color_enabled(): - env_addons['NOSE_OPENSTACK_COLOR'] = '0' - if self.get_bool_option("verbose", default_value=True): - env_addons['NOSE_OPENSTACK_STDOUT'] = '1' - return env_addons - - def run_tests(self): - app_dir = self.get_option('app_dir') - if not sh.isdir(app_dir): - LOG.warn("Unable to find application directory at %s, can not run %s tests.", - colorizer.quote(app_dir), colorizer.quote(self.name)) - return - cmd = self._get_test_command() - env = self._get_env() - with open(os.devnull, 'wb') as null_fh: - if self.get_bool_option("verbose", default_value=False): - null_fh = None - try: - sh.execute(*cmd, stdout_fh=None, stderr_fh=null_fh, cwd=app_dir, env_overrides=env) - except excp.ProcessExecutionError as e: - if self.get_bool_option("ignore-test-failures", default_value=False): - LOG.warn("Ignoring test failure of component %s: %s", colorizer.quote(self.name), e) - else: - raise e - - def show_coverage(self): - app_dir = self.get_option('app_dir') - if not sh.isdir(app_dir): - LOG.warn("Unable to find application directory at %s, can not run %s tests.", - colorizer.quote(app_dir), colorizer.quote(self.name)) - return - env = self._get_env() - # Try to combine reports - cmd = ['coverage', 'combine'] - try: - sh.execute(*cmd, stdout_fh=None, stderr_fh=None, cwd=app_dir, env_overrides=env) - except excp.ProcessExecutionError as e: - pass - - cmd = ['coverage', 'report'] - try: - sh.execute(*cmd, stdout_fh=None, stderr_fh=None, cwd=app_dir, env_overrides=env) - except excp.ProcessExecutionError as e: - LOG.warn("Something wrong with %s: %s", colorizer.quote(self.name), e) - - -#### -#### PACKAGING CLASSES -#### - -class EmptyPackagingComponent(component.Component): - def package(self): - return None diff --git a/anvil/component.py b/anvil/components/base.py similarity index 100% rename from anvil/component.py rename to anvil/components/base.py diff --git a/anvil/components/base_install.py b/anvil/components/base_install.py new file mode 100644 index 00000000..48dadca7 --- /dev/null +++ b/anvil/components/base_install.py @@ -0,0 +1,626 @@ +# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from anvil import colorizer +from anvil import decorators +from anvil import downloader as down +from anvil import exceptions as excp +from anvil import importer +from anvil import log as logging +from anvil import patcher +from anvil import shell as sh +from anvil import trace as tr +from anvil import utils + +from anvil.packaging import pip + +from anvil.packaging.helpers import pip_helper + +from anvil.components import base +from anvil.components.configurators import base as conf + +import re + +LOG = logging.getLogger(__name__) + +# Cache of accessed packagers +_PACKAGERS = {} + + +def make_packager(package, default_class, **kwargs): + packager_name = package.get('packager_name') or '' + packager_name = packager_name.strip() + if packager_name: + packager_cls = importer.import_entry_point(packager_name) + else: + packager_cls = default_class + if packager_cls in _PACKAGERS: + return _PACKAGERS[packager_cls] + p = packager_cls(**kwargs) + _PACKAGERS[packager_cls] = p + return p + + +# Remove any private keys from a package dictionary +def filter_package(pkg): + n_pkg = {} + for (k, v) in pkg.items(): + if not k or k.startswith("_"): + continue + else: + n_pkg[k] = v + return n_pkg + +class EmptyPackagingComponent(base.Component): + def package(self): + return None + + +class PkgUninstallComponent(base.Component): + def __init__(self, *args, **kargs): + base.Component.__init__(self, *args, **kargs) + trace_fn = tr.trace_filename(self.get_option('trace_dir'), 'created') + self.tracereader = tr.TraceReader(trace_fn) + self.purge_packages = kargs.get('purge_packages') + + def unconfigure(self): + self._unconfigure_links() + + def _unconfigure_links(self): + sym_files = self.tracereader.symlinks_made() + if sym_files: + utils.log_iterable(sym_files, logger=LOG, + header="Removing %s symlink files" % (len(sym_files))) + for fn in sym_files: + sh.unlink(fn, run_as_root=True) + + def uninstall(self): + self._uninstall_pkgs() + self._uninstall_files() + + def post_uninstall(self): + self._uninstall_dirs() + + def pre_uninstall(self): + pass + + def _uninstall_pkgs(self): + pkgs = self.tracereader.packages_installed() + if pkgs: + pkg_names = set([p['name'] for p in pkgs]) + utils.log_iterable(pkg_names, logger=LOG, + header="Potentially removing %s distribution packages" % (len(pkg_names))) + which_removed = [] + with utils.progress_bar('Uninstalling', len(pkgs), reverse=True) as p_bar: + for (i, p) in enumerate(pkgs): + uninstaller = make_packager(p, self.distro.package_manager_class, + distro=self.distro, + remove_default=self.purge_packages) + if uninstaller.remove(p): + which_removed.append(p['name']) + p_bar.update(i + 1) + utils.log_iterable(which_removed, logger=LOG, + header="Actually removed %s distribution packages" % (len(which_removed))) + + def _uninstall_files(self): + files_touched = self.tracereader.files_touched() + if files_touched: + utils.log_iterable(files_touched, logger=LOG, + header="Removing %s miscellaneous files" % (len(files_touched))) + for fn in files_touched: + sh.unlink(fn, run_as_root=True) + + def _uninstall_dirs(self): + dirs_made = self.tracereader.dirs_made() + dirs_alive = filter(sh.isdir, dirs_made) + if dirs_alive: + utils.log_iterable(dirs_alive, logger=LOG, + header="Removing %s created directories" % (len(dirs_alive))) + for dir_name in dirs_alive: + sh.deldir(dir_name, run_as_root=True) + + +class PythonUninstallComponent(PkgUninstallComponent): + + def uninstall(self): + self._uninstall_python() + self._uninstall_pips() + PkgUninstallComponent.uninstall(self) + + def _uninstall_pips(self): + pips = self.tracereader.pips_installed() + if pips: + pip_names = set([p['name'] for p in pips]) + utils.log_iterable(pip_names, logger=LOG, + header="Potentially removing %s python packages" % (len(pip_names))) + which_removed = [] + with utils.progress_bar('Uninstalling', len(pips), reverse=True) as p_bar: + for (i, p) in enumerate(pips): + try: + uninstaller = make_packager(p, pip.Packager, + distro=self.distro, + remove_default=self.purge_packages) + if uninstaller.remove(p): + which_removed.append(p['name']) + except excp.ProcessExecutionError as e: + # NOTE(harlowja): pip seems to die if a pkg isn't there even in quiet mode + combined = (str(e.stderr) + str(e.stdout)) + if not re.search(r"not\s+installed", combined, re.I): + raise + p_bar.update(i + 1) + utils.log_iterable(which_removed, logger=LOG, + header="Actually removed %s python packages" % (len(which_removed))) + + def _uninstall_python(self): + py_listing = self.tracereader.py_listing() + if py_listing: + py_listing_dirs = set() + for (_name, where) in py_listing: + py_listing_dirs.add(where) + utils.log_iterable(py_listing_dirs, logger=LOG, + header="Uninstalling %s python setups" % (len(py_listing_dirs))) + unsetup_cmd = self.distro.get_command('python', 'unsetup') + for where in py_listing_dirs: + if sh.isdir(where): + sh.execute(*unsetup_cmd, cwd=where, run_as_root=True) + else: + LOG.warn("No python directory found at %s - skipping", colorizer.quote(where, quote_color='red')) + + +class PkgInstallComponent(base.Component): + def __init__(self, *args, **kargs): + base.Component.__init__(self, *args, **kargs) + trace_fn = tr.trace_filename(self.get_option('trace_dir'), 'created') + self.tracewriter = tr.TraceWriter(trace_fn, break_if_there=False) + self.configurator = conf.Configurator(self) + + def _get_download_config(self): + return None + + def _get_download_location(self): + key = self._get_download_config() + if not key: + return (None, None) + uri = self.get_option(key, default_value='').strip() + if not uri: + raise ValueError(("Could not find uri in config to download " + "from option %s") % (key)) + return (uri, self.get_option('app_dir')) + + def download(self): + (from_uri, target_dir) = self._get_download_location() + if not from_uri and not target_dir: + return [] + else: + uris = [from_uri] + utils.log_iterable(uris, logger=LOG, + header="Downloading from %s uris" % (len(uris))) + sh.mkdirslist(target_dir, tracewriter=self.tracewriter) + # This is used to delete what is downloaded (done before + # fetching to ensure its cleaned up even on download failures) + self.tracewriter.download_happened(target_dir, from_uri) + fetcher = down.GitDownloader(self.distro, from_uri, target_dir) + fetcher.download() + return uris + + def patch(self, section): + what_patches = self.get_option('patches', section) + (_from_uri, target_dir) = self._get_download_location() + if not what_patches: + what_patches = [] + canon_what_patches = [] + for path in what_patches: + if sh.isdir(path): + canon_what_patches.extend(sorted(sh.listdir(path, files_only=True))) + elif sh.isfile(path): + canon_what_patches.append(path) + if canon_what_patches: + patcher.apply_patches(canon_what_patches, target_dir) + + def config_params(self, config_fn): + mp = dict(self.params) + if config_fn: + mp['CONFIG_FN'] = config_fn + return mp + + @property + def packages(self): + pkg_list = self.get_option('packages', default_value=[]) + if not pkg_list: + pkg_list = [] + for name, values in self.subsystems.items(): + if 'packages' in values: + LOG.debug("Extending package list with packages for subsystem: %r", name) + pkg_list.extend(values.get('packages')) + return pkg_list + + def install(self): + LOG.debug('Preparing to install packages for: %r', self.name) + pkgs = self.packages + if pkgs: + pkg_names = set([p['name'] for p in pkgs]) + utils.log_iterable(pkg_names, logger=LOG, + header="Setting up %s distribution packages" % (len(pkg_names))) + with utils.progress_bar('Installing', len(pkgs)) as p_bar: + for (i, p) in enumerate(pkgs): + installer = make_packager(p, self.distro.package_manager_class, + distro=self.distro) + installer.install(p) + # Mark that this happened so that we can uninstall it + self.tracewriter.package_installed(filter_package(p)) + p_bar.update(i + 1) + + def pre_install(self): + pkgs = self.packages + for p in pkgs: + installer = make_packager(p, self.distro.package_manager_class, + distro=self.distro) + installer.pre_install(p, self.params) + + def post_install(self): + pkgs = self.packages + for p in pkgs: + installer = make_packager(p, self.distro.package_manager_class, + distro=self.distro) + installer.post_install(p, self.params) + + def _configure_files(self): + config_fns = self.configurator.config_files + if config_fns: + utils.log_iterable(config_fns, logger=LOG, + header="Configuring %s files" % (len(config_fns))) + for fn in config_fns: + tgt_fn = self.configurator.target_config(fn) + sh.mkdirslist(sh.dirname(tgt_fn), tracewriter=self.tracewriter) + (source_fn, contents) = self.configurator.source_config(fn) + LOG.debug("Configuring file %s ---> %s.", (source_fn), (tgt_fn)) + contents = self.configurator.config_param_replace(fn, contents, self.config_params(fn)) + contents = self.configurator.config_adjust(contents, fn) + sh.write_file(tgt_fn, contents, tracewriter=self.tracewriter) + return len(config_fns) + + def _configure_symlinks(self): + links = self.configurator.symlinks + if not links: + return 0 + # This sort happens so that we link in the correct order + # although it might not matter. Either way. We ensure that the right + # order happens. Ie /etc/blah link runs before /etc/blah/blah + link_srcs = sorted(links.keys()) + link_srcs.reverse() + link_nice = [] + for source in link_srcs: + links_to_be = links[source] + for link in links_to_be: + link_nice.append("%s => %s" % (link, source)) + utils.log_iterable(link_nice, logger=LOG, + header="Creating %s sym-links" % (len(link_nice))) + links_made = 0 + for source in link_srcs: + links_to_be = links[source] + for link in links_to_be: + try: + LOG.debug("Symlinking %s to %s.", link, source) + sh.symlink(source, link, tracewriter=self.tracewriter) + links_made += 1 + except (IOError, OSError) as e: + LOG.warn("Symlinking %s to %s failed: %s", colorizer.quote(link), colorizer.quote(source), e) + return links_made + + def configure(self): + return self._configure_files() + self._configure_symlinks() + + +class PythonInstallComponent(PkgInstallComponent): + def __init__(self, *args, **kargs): + PkgInstallComponent.__init__(self, *args, **kargs) + self.requires_files = [ + sh.joinpths(self.get_option('app_dir'), 'tools', 'pip-requires'), + ] + if self.get_bool_option('use_tests_requires', default_value=True): + self.requires_files.append(sh.joinpths(self.get_option('app_dir'), 'tools', 'test-requires')) + + def _get_download_config(self): + return 'get_from' + + @property + def python_directories(self): + py_dirs = {} + app_dir = self.get_option('app_dir') + if sh.isdir(app_dir): + py_dirs[self.name] = app_dir + return py_dirs + + @property + def packages(self): + pkg_list = super(PythonInstallComponent, self).packages + if not pkg_list: + pkg_list = [] + pkg_list.extend(self._get_mapped_packages()) + return pkg_list + + @property + def pips_to_packages(self): + pip_pkg_list = self.get_option('pip_to_package', default_value=[]) + if not pip_pkg_list: + pip_pkg_list = [] + return pip_pkg_list + + @property + def pip_requires(self): + all_pips = [] + for fn in self.requires_files: + all_pips.extend(self._extract_pip_requires(fn)) + return all_pips + + def _match_pip_requires(self, pip_req): + + def pip_use(who, there_pip): + if there_pip.key != pip_req.key: + return False + if not len(pip_req.specs): + # No version/restrictions specified + return True + there_version = None + if not there_pip.specs or there_pip == pip_req: + return True + # Different possibly incompat. versions found... + if there_version is None: + # Assume pip will install the correct version anyway + if who != self.name: + msg = ("Component %r asked for package '%s'" + " and '%s' is being selected from %r instead...") + LOG.debug(msg, self.name, pip_req, there_pip, who) + return True + else: + if who != self.name: + msg = ("Component %r provides package '%s'" + " but '%s' is being asked for by %r instead...") + LOG.warn(msg, who, there_pip, pip_req, self.name) + return False + + LOG.debug("Attempting to find who satisfies pip requirement '%s'", pip_req) + + # Try to find it in anyones pip -> pkg list + all_pip_2_pkgs = { + self.name: self.pips_to_packages, + } + # Gather them all (but only if they activate before me) + # since if they activate after, we can't depend on it + # to satisfy our requirement... + for (name, c) in self.instances.items(): + if c is self or not c.activated: + continue + if isinstance(c, (PythonInstallComponent)): + all_pip_2_pkgs[name] = c.pips_to_packages + for (who, pips_2_pkgs) in all_pip_2_pkgs.items(): + for pip_info in pips_2_pkgs: + there_pip = pip.extract_requirement(pip_info) + if not pip_use(who, there_pip): + continue + LOG.debug("Matched pip->pkg '%s' from component %r", there_pip, who) + return (dict(pip_info.get('package')), False) + + # Ok nobody had it in a pip->pkg mapping + # but see if they had it in there pip collection + all_pips = { + self.name: self._base_pips(), # Use base pips to avoid recursion... + } + for (name, c) in self.instances.items(): + if not c.activated or c is self: + continue + if isinstance(c, (PythonInstallComponent)): + all_pips[name] = c._base_pips() # pylint: disable=W0212 + for (who, there_pips) in all_pips.items(): + for pip_info in there_pips: + there_pip = pip.extract_requirement(pip_info) + if not pip_use(who, there_pip): + continue + LOG.debug("Matched pip '%s' from component %r", there_pip, who) + return (dict(pip_info), True) + + # Ok nobody had it in there pip->pkg mapping or pip mapping + # but now lets see if we can automatically find + # a pip->pkg mapping for them using the good ole' + # rpm/yum database. + installer = make_packager({}, self.distro.package_manager_class, + distro=self.distro) + + # TODO(harlowja): make this better + if installer and hasattr(installer, 'match_pip_2_package'): + try: + dist_pkg = installer.match_pip_2_package(pip_req) + if dist_pkg: + pkg_info = { + 'name': str(dist_pkg.name), + 'version': str(dist_pkg.version), + '__requirement': dist_pkg, + } + LOG.debug("Auto-matched (dist) %s -> %s", pip_req, dist_pkg) + return (pkg_info, False) + except excp.DependencyException as e: + LOG.warn("Unable to automatically map pip to package: %s", e) + + # Ok still nobody has it, search pypi... + pypi_pkg = pip_helper.find_pypi_match(pip_req) + if pypi_pkg: + pkg_info = { + 'name': str(pypi_pkg.key), + '__requirement': pypi_pkg, + } + try: + pkg_info['version'] = pypi_pkg.specs[0][1] + except IndexError: + pass + LOG.debug("Auto-matched (pypi) %s -> %s", pip_req, pypi_pkg) + return (pkg_info, True) + + return (None, False) + + def _get_mapped_packages(self): + add_on_pkgs = [] + all_pips = self.pip_requires + for details in all_pips: + pkg_info = details['package'] + from_pip = details['from_pip'] + if from_pip or not pkg_info: + continue + # Keep the initial requirement + pkg_info = dict(pkg_info) + pkg_info['__requirement'] = details['requirement'] + add_on_pkgs.append(pkg_info) + return add_on_pkgs + + def _get_mapped_pips(self): + add_on_pips = [] + all_pips = self.pip_requires + for details in all_pips: + pkg_info = details['package'] + from_pip = details['from_pip'] + if not from_pip or not pkg_info: + continue + # Keep the initial requirement + pkg_info = dict(pkg_info) + pkg_info['__requirement'] = details['requirement'] + add_on_pips.append(pkg_info) + return add_on_pips + + def _base_pips(self): + pip_list = self.get_option('pips', default_value=[]) + if not pip_list: + pip_list = [] + for (name, values) in self.subsystems.items(): + if 'pips' in values: + LOG.debug("Extending pip list with pips for subsystem: %r" % (name)) + pip_list.extend(values.get('pips')) + return pip_list + + @property + def pips(self): + pip_list = self._base_pips() + pip_list.extend(self._get_mapped_pips()) + return pip_list + + def _install_pips(self): + pips = self.pips + if pips: + pip_names = set([p['name'] for p in pips]) + utils.log_iterable(pip_names, logger=LOG, + header="Setting up %s python packages" % (len(pip_names))) + with utils.progress_bar('Installing', len(pips)) as p_bar: + for (i, p) in enumerate(pips): + installer = make_packager(p, pip.Packager, + distro=self.distro) + installer.install(p) + # Note that we did it so that we can remove it... + self.tracewriter.pip_installed(filter_package(p)) + p_bar.update(i + 1) + + def _clean_pip_requires(self): + # Fixup these files if they exist, sometimes they have 'junk' in them + # that anvil will install instead of pip or setup.py and we don't want + # the setup.py file to attempt to install said dependencies since it + # typically picks locations that either are not what we desire or if + # said file contains editables, it may even pick external source directories + # which is what anvil is setting up as well... + req_fns = [f for f in self.requires_files if sh.isfile(f)] + if req_fns: + utils.log_iterable(req_fns, logger=LOG, + header="Adjusting %s pip 'requires' files" % (len(req_fns))) + for fn in req_fns: + old_lines = sh.load_file(fn).splitlines() + new_lines = self._filter_pip_requires(fn, old_lines) + contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(), "\n".join(new_lines)) + sh.write_file_and_backup(fn, contents) + return len(req_fns) + + def _filter_pip_requires(self, fn, lines): + # The default does no filtering except to ensure that said lines are valid... + return lines + + def pre_install(self): + self._verify_pip_requires() + PkgInstallComponent.pre_install(self) + for p in self.pips: + installer = make_packager(p, pip.Packager, + distro=self.distro) + installer.pre_install(p, self.params) + + def post_install(self): + PkgInstallComponent.post_install(self) + for p in self.pips: + installer = make_packager(p, pip.Packager, + distro=self.distro) + installer.post_install(p, self.params) + + def _install_python_setups(self): + py_dirs = self.python_directories + if py_dirs: + real_dirs = {} + for (name, wkdir) in py_dirs.items(): + real_dirs[name] = wkdir + if not real_dirs[name]: + real_dirs[name] = self.get_option('app_dir') + utils.log_iterable(real_dirs.values(), logger=LOG, + header="Setting up %s python directories" % (len(real_dirs))) + setup_cmd = self.distro.get_command('python', 'setup') + for (name, working_dir) in real_dirs.items(): + sh.mkdirslist(working_dir, tracewriter=self.tracewriter) + setup_fn = sh.joinpths(self.get_option('trace_dir'), "%s.python.setup" % (name)) + sh.execute(*setup_cmd, cwd=working_dir, run_as_root=True, + stderr_fn='%s.stderr' % (setup_fn), + stdout_fn='%s.stdout' % (setup_fn), + tracewriter=self.tracewriter) + self.tracewriter.py_installed(name, working_dir) + + def _python_install(self): + self._install_pips() + self._install_python_setups() + + @decorators.memoized + def _extract_pip_requires(self, fn): + if not sh.isfile(fn): + return [] + LOG.debug("Resolving dependencies from %s.", colorizer.quote(fn)) + pips_needed = pip_helper.parse_requirements(sh.load_file(fn)) + matchings = [] + for req in pips_needed: + (pkg_info, from_pip) = self._match_pip_requires(req) + matchings.append({ + 'requirement': req, + 'package': pkg_info, + 'from_pip': from_pip, + 'needed_by': fn, + }) + return matchings + + def _verify_pip_requires(self): + all_pips = self.pip_requires + for details in all_pips: + req = details['requirement'] + needed_by = details['needed_by'] + pkg_info = details['package'] + if not pkg_info: + raise excp.DependencyException(("Pip dependency '%s' needed by '%s' is not translatable to a listed" + " (from this or previously activated components) pip package" + ' or a pip->package mapping!') % (req, needed_by)) + + def install(self): + PkgInstallComponent.install(self) + self._python_install() + + def configure(self): + configured_am = PkgInstallComponent.configure(self) + configured_am += self._clean_pip_requires() + return configured_am diff --git a/anvil/components/base_runtime.py b/anvil/components/base_runtime.py new file mode 100644 index 00000000..7b41064d --- /dev/null +++ b/anvil/components/base_runtime.py @@ -0,0 +1,266 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from anvil import colorizer +from anvil import exceptions as excp +from anvil import importer +from anvil import log as logging +from anvil import shell as sh +from anvil import trace as tr +from anvil import utils + +from anvil.components import base + +LOG = logging.getLogger(__name__) + +DEFAULT_RUNNER = 'anvil.runners.fork:ForkRunner' + +#### +#### STATUS CONSTANTS +#### +STATUS_INSTALLED = 'installed' +STATUS_STARTED = "started" +STATUS_STOPPED = "stopped" +STATUS_UNKNOWN = "unknown" + + +class ProgramStatus(object): + def __init__(self, status, name=None, details=''): + self.name = name + self.status = status + self.details = details + + +class Program(object): + def __init__(self, name, path=None, working_dir=None, argv=None): + self.name = name + if path is None: + self.path = name + else: + self.path = path + self.working_dir = working_dir + if argv is None: + self.argv = tuple() + else: + self.argv = tuple(argv) + + def __str__(self): + what = str(self.name) + if self.path: + what += " (%s)" % (self.path) + return what + + +class ProgramRuntime(base.Component): + @property + def applications(self): + # A list of applications since a single component sometimes + # has a list of programs to start (ie nova) instead of a single application (ie the db) + return [] + + def restart(self): + # How many applications restarted + return 0 + + def post_start(self): + pass + + def pre_start(self): + pass + + def statii(self): + # A list of statuses since a single component sometimes + # has a list of programs to report on (ie nova) instead of a single application (ie the db) + return [] + + def start(self): + # How many applications started + return 0 + + def stop(self): + # How many applications stopped + return 0 + + # TODO(harlowja): seems like this could be a mixin? + def wait_active(self, between_wait=1, max_attempts=5): + # Attempt to wait until all potentially started applications + # are actually started (for whatever defintion of started is applicable) + # for up to a given amount of attempts and wait time between attempts. + num_started = len(self.applications) + if not num_started: + raise excp.StatusException("No %r programs started, can not wait for them to become active..." % (self.name)) + + def waiter(try_num): + LOG.info("Waiting %s seconds for component %s programs to start.", between_wait, colorizer.quote(self.name)) + LOG.info("Please wait...") + sh.sleep(between_wait) + + for i in range(0, max_attempts): + statii = self.statii() + if len(statii) >= num_started: # >= if someone reports more than started... + not_worked = [] + for p in statii: + if p.status != STATUS_STARTED: + not_worked.append(p) + if len(not_worked) == 0: + return + else: + # Eck less applications were found with status then what were started! + LOG.warn("%s less applications reported status than were actually started!", + num_started - len(statii)) + waiter(i + 1) + + tot_time = max(0, (between_wait * max_attempts)) + raise excp.StatusException("Failed waiting %s seconds for component %r programs to become active..." + % (tot_time, self.name)) + + +class EmptyRuntime(ProgramRuntime): + pass + + +class PythonRuntime(ProgramRuntime): + def __init__(self, *args, **kargs): + ProgramRuntime.__init__(self, *args, **kargs) + start_trace = tr.trace_filename(self.get_option('trace_dir'), 'start') + self.tracewriter = tr.TraceWriter(start_trace, break_if_there=True) + self.tracereader = tr.TraceReader(start_trace) + + def app_params(self, program): + params = dict(self.params) + if program and program.name: + params['APP_NAME'] = str(program.name) + return params + + def start(self): + # Perform a check just to make sure said programs aren't already started and bail out + # so that it we don't unintentionally start new ones and thus causing confusion for all + # involved... + what_may_already_be_started = [] + try: + what_may_already_be_started = self.tracereader.apps_started() + except excp.NoTraceException: + pass + if what_may_already_be_started: + msg = "%s programs of component %s may already be running, did you forget to stop those?" + raise excp.StartException(msg % (len(what_may_already_be_started), self.name)) + + # Select how we are going to start it and get on with the show... + runner_entry_point = self.get_option("run_type", default_value=DEFAULT_RUNNER) + starter_args = [self, runner_entry_point] + starter = importer.construct_entry_point(runner_entry_point, *starter_args) + amount_started = 0 + for program in self.applications: + self._start_app(program, starter) + amount_started += 1 + return amount_started + + def _start_app(self, program, starter): + app_working_dir = program.working_dir + if not app_working_dir: + app_working_dir = self.get_option('app_dir') + + # Un-templatize whatever argv (program options) the program has specified + # with whatever program params were retrieved to create the 'real' set + # of program options (if applicable) + app_params = self.app_params(program) + if app_params: + app_argv = [utils.expand_template(arg, app_params) for arg in program.argv] + else: + app_argv = program.argv + LOG.debug("Starting %r using a %r", program.name, starter) + + # TODO(harlowja): clean this function params up (should just take a program) + details_path = starter.start(program.name, + app_pth=program.path, + app_dir=app_working_dir, + opts=app_argv) + + # This trace is used to locate details about what/how to stop + LOG.info("Started program %s under component %s.", colorizer.quote(program.name), self.name) + self.tracewriter.app_started(program.name, details_path, starter.name) + + def _locate_investigators(self, applications_started): + # Recreate the runners that can be used to dive deeper into the applications list + # that was started (a 3 tuple of (name, trace, who_started)). + investigators_created = {} + to_investigate = [] + for (name, _trace, who_started) in applications_started: + investigator = investigators_created.get(who_started) + if investigator is None: + try: + investigator_args = [self, who_started] + investigator = importer.construct_entry_point(who_started, *investigator_args) + investigators_created[who_started] = investigator + except RuntimeError as e: + LOG.warn("Could not load class %s which should be used to investigate %s: %s", + colorizer.quote(who_started), colorizer.quote(name), e) + continue + to_investigate.append((name, investigator)) + return to_investigate + + def stop(self): + # Anything to stop in the first place?? + what_was_started = [] + try: + what_was_started = self.tracereader.apps_started() + except excp.NoTraceException: + pass + if not what_was_started: + return 0 + + # Get the investigators/runners which can be used + # to actually do the stopping and attempt to perform said stop. + applications_stopped = [] + for (name, handler) in self._locate_investigators(what_was_started): + handler.stop(name) + applications_stopped.append(name) + if applications_stopped: + utils.log_iterable(applications_stopped, + header="Stopped %s programs started under %s component" % (len(applications_stopped), self.name), + logger=LOG) + + # Only if we stopped the amount which was supposedly started can + # we actually remove the trace where those applications have been + # marked as started in (ie the connection back to how they were started) + if len(applications_stopped) < len(what_was_started): + diff = len(what_was_started) - len(applications_stopped) + LOG.warn(("%s less applications were stopped than were started, please check out %s" + " to stop these program manually."), diff, colorizer.quote(self.tracereader.filename(), quote_color='yellow')) + else: + sh.unlink(self.tracereader.filename()) + + return len(applications_stopped) + + def statii(self): + # Anything to get status on in the first place?? + what_was_started = [] + try: + what_was_started = self.tracereader.apps_started() + except excp.NoTraceException: + pass + if not what_was_started: + return [] + + # Get the investigators/runners which can be used + # to actually do the status inquiry and attempt to perform said inquiry. + statii = [] + for (name, handler) in self._locate_investigators(what_was_started): + (status, details) = handler.status(name) + statii.append(ProgramStatus(name=name, + status=status, + details=details)) + return statii diff --git a/anvil/components/base_testing.py b/anvil/components/base_testing.py new file mode 100644 index 00000000..1e7e454b --- /dev/null +++ b/anvil/components/base_testing.py @@ -0,0 +1,143 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +from anvil import cfg +from anvil import colorizer +from anvil import exceptions as excp +from anvil import log as logging +from anvil import shell as sh +from anvil import utils + +from anvil.components import base +from anvil.components import base_install as binstall +from anvil.packaging.helpers import pip_helper + +LOG = logging.getLogger(__name__) + +class EmptyTestingComponent(base.Component): + def run_tests(self): + return + + +class PythonTestingComponent(base.Component): + def __init__(self, *args, **kargs): + base.Component.__init__(self, *args, **kargs) + self.helper = pip_helper.Helper(self.distro) + + def _get_test_exclusions(self): + return self.get_option('exclude_tests', default_value=[]) + + def _use_run_tests(self): + return True + + def _get_test_command(self): + # See: http://docs.openstack.org/developer/nova/devref/unit_tests.html + # And: http://wiki.openstack.org/ProjectTestingInterface + app_dir = self.get_option('app_dir') + if sh.isfile(sh.joinpths(app_dir, 'run_tests.sh')) and self._use_run_tests(): + cmd = [sh.joinpths(app_dir, 'run_tests.sh'), '-N'] + if not self._use_pep8(): + cmd.append('--no-pep8') + else: + # Assume tox is being used, which we can't use directly + # since anvil doesn't really do venv stuff (its meant to avoid those...) + cmd = ['nosetests'] + # See: $ man nosetests + if self.get_bool_option("verbose", default_value=False): + cmd.append('--nologcapture') + for e in self._get_test_exclusions(): + cmd.append('--exclude=%s' % (e)) + xunit_fn = self.get_option("xunit_filename") + if xunit_fn: + cmd.append("--with-xunit") + cmd.append("--xunit-file=%s" % (xunit_fn)) + return cmd + + def _use_pep8(self): + # Seems like the varying versions are borking pep8 from working... + i_sibling = self.siblings.get('install') + # Check if whats installed actually matches + pep8_wanted = None + if isinstance(i_sibling, (binstall.PythonInstallComponent)): + for p in i_sibling.pip_requires: + req = p['requirement'] + if req.key == "pep8": + pep8_wanted = req + break + if not pep8_wanted: + # Doesn't matter since its not wanted anyway + return True + pep8_there = self.helper.get_installed('pep8') + if not pep8_there: + # Hard to use it if it isn't there... + LOG.warn("Pep8 version mismatch, none is installed but %s is wanting %s", + self.name, pep8_wanted) + return False + if not (pep8_there == pep8_wanted): + # Versions not matching, this is causes pep8 to puke when it doesn't need to + # so skip it from running in the first place... + LOG.warn("Pep8 version mismatch, installed is %s but %s is applying %s", + pep8_there, self.name, pep8_wanted) + return False + return self.get_bool_option('use_pep8', default_value=True) + + def _get_env(self): + env_addons = {} + tox_fn = sh.joinpths(self.get_option('app_dir'), 'tox.ini') + if sh.isfile(tox_fn): + # Suck out some settings from the tox file + try: + tox_cfg = cfg.BuiltinConfigParser(fns=[tox_fn]) + env_values = tox_cfg.get('testenv', 'setenv') or '' + for env_line in env_values.splitlines(): + env_line = env_line.strip() + env_line = env_line.split("#")[0].strip() + if not env_line: + continue + env_entry = env_line.split('=', 1) + if len(env_entry) == 2: + (name, value) = env_entry + name = name.strip() + value = value.strip() + if name.lower() != 'virtual_env': + env_addons[name] = value + if env_addons: + LOG.debug("From %s we read in %s environment settings:", tox_fn, len(env_addons)) + utils.log_object(env_addons, logger=LOG, level=logging.DEBUG) + except IOError: + pass + return env_addons + + def run_tests(self): + app_dir = self.get_option('app_dir') + if not sh.isdir(app_dir): + LOG.warn("Unable to find application directory at %s, can not run %s tests.", + colorizer.quote(app_dir), colorizer.quote(self.name)) + return + cmd = self._get_test_command() + env = self._get_env() + with open(os.devnull, 'wb') as null_fh: + if self.get_bool_option("verbose", default_value=False): + null_fh = None + try: + sh.execute(*cmd, stdout_fh=None, stderr_fh=null_fh, cwd=app_dir, env_overrides=env) + except excp.ProcessExecutionError as e: + if self.get_bool_option("ignore-test-failures", default_value=False): + LOG.warn("Ignoring test failure of component %s: %s", colorizer.quote(self.name), e) + else: + raise e diff --git a/anvil/components/cinder.py b/anvil/components/cinder.py index 56dd0f84..4fdbb11c 100644 --- a/anvil/components/cinder.py +++ b/anvil/components/cinder.py @@ -15,11 +15,13 @@ # under the License. from anvil import colorizer -from anvil import components as comp from anvil import log as logging from anvil import shell as sh from anvil import utils +from anvil.components import base_install as binstall +from anvil.components import base_runtime as bruntime + from anvil.components.configurators import cinder as cconf LOG = logging.getLogger(__name__) @@ -31,20 +33,20 @@ SYNC_DB_CMD = [sh.joinpths('$BIN_DIR', 'cinder-manage'), BIN_DIR = 'bin' -class CinderUninstaller(comp.PythonUninstallComponent): +class CinderUninstaller(binstall.PythonUninstallComponent): def __init__(self, *args, **kargs): - comp.PythonUninstallComponent.__init__(self, *args, **kargs) + binstall.PythonUninstallComponent.__init__(self, *args, **kargs) self.bin_dir = sh.joinpths(self.get_option('app_dir'), BIN_DIR) -class CinderInstaller(comp.PythonInstallComponent): +class CinderInstaller(binstall.PythonInstallComponent): def __init__(self, *args, **kargs): - comp.PythonInstallComponent.__init__(self, *args, **kargs) + binstall.PythonInstallComponent.__init__(self, *args, **kargs) self.bin_dir = sh.joinpths(self.get_option('app_dir'), BIN_DIR) self.configurator = cconf.CinderConfigurator(self) def post_install(self): - comp.PythonInstallComponent.post_install(self) + binstall.PythonInstallComponent.post_install(self) if self.get_bool_option('db-sync'): self.configurator.setup_db() self._sync_db() @@ -62,14 +64,14 @@ class CinderInstaller(comp.PythonInstallComponent): utils.execute_template(*cmds, cwd=self.bin_dir, params=self.config_params(None)) def config_params(self, config_fn): - mp = comp.PythonInstallComponent.config_params(self, config_fn) + mp = binstall.PythonInstallComponent.config_params(self, config_fn) mp['BIN_DIR'] = sh.joinpths(self.get_option('app_dir'), BIN_DIR) return mp -class CinderRuntime(comp.PythonRuntime): +class CinderRuntime(bruntime.PythonRuntime): def __init__(self, *args, **kargs): - comp.PythonRuntime.__init__(self, *args, **kargs) + bruntime.PythonRuntime.__init__(self, *args, **kargs) self.bin_dir = sh.joinpths(self.get_option('app_dir'), BIN_DIR) self.config_path = sh.joinpths(self.get_option('cfg_dir'), cconf.API_CONF) @@ -80,11 +82,11 @@ class CinderRuntime(comp.PythonRuntime): name = "cinder-%s" % (name.lower()) path = sh.joinpths(self.bin_dir, name) if sh.is_executable(path): - apps.append(comp.Program(name, path, argv=self._fetch_argv(name))) + apps.append(bruntime.Program(name, path, argv=self._fetch_argv(name))) return apps def app_params(self, program): - params = comp.PythonRuntime.app_params(self, program) + params = bruntime.PythonRuntime.app_params(self, program) params['CFG_FILE'] = self.config_path return params diff --git a/anvil/components/db.py b/anvil/components/db.py index ad62f642..219e274a 100644 --- a/anvil/components/db.py +++ b/anvil/components/db.py @@ -15,11 +15,13 @@ # under the License. from anvil import colorizer -from anvil import components as comp from anvil import log as logging from anvil import shell as sh from anvil import utils +from anvil.components import base_install as binstall +from anvil.components import base_runtime as bruntime + from anvil.components.helpers import db as dbhelper import abc @@ -34,10 +36,10 @@ RESET_BASE_PW = '' BASE_ERROR = dbhelper.BASE_ERROR -class DBUninstaller(comp.PkgUninstallComponent): +class DBUninstaller(binstall.PkgUninstallComponent): def __init__(self, *args, **kargs): - comp.PkgUninstallComponent.__init__(self, *args, **kargs) + binstall.PkgUninstallComponent.__init__(self, *args, **kargs) self.runtime = self.siblings.get('running') def warm_configs(self): @@ -67,17 +69,17 @@ class DBUninstaller(comp.PkgUninstallComponent): "reset the password to %s before the next install"), colorizer.quote(RESET_BASE_PW)) -class DBInstaller(comp.PkgInstallComponent): +class DBInstaller(binstall.PkgInstallComponent): __meta__ = abc.ABCMeta def __init__(self, *args, **kargs): - comp.PkgInstallComponent.__init__(self, *args, **kargs) + binstall.PkgInstallComponent.__init__(self, *args, **kargs) self.runtime = self.siblings.get('running') def config_params(self, config_fn): # This dictionary will be used for parameter replacement # In pre-install and post-install sections - mp = comp.PkgInstallComponent.config_params(self, config_fn) + mp = binstall.PkgInstallComponent.config_params(self, config_fn) mp.update({ 'PASSWORD': dbhelper.get_shared_passwords(self)['pw'], 'BOOT_START': "true", @@ -95,7 +97,7 @@ class DBInstaller(comp.PkgInstallComponent): pass def post_install(self): - comp.PkgInstallComponent.post_install(self) + binstall.PkgInstallComponent.post_install(self) # Fix up the db configs self._configure_db_confs() @@ -133,7 +135,7 @@ class DBInstaller(comp.PkgInstallComponent): **dbhelper.get_shared_passwords(self)) -class DBRuntime(comp.ProgramRuntime): +class DBRuntime(bruntime.ProgramRuntime): def _get_command(self, action): db_type = self.get_option("type") distro_options = self.distro.get_command_config(db_type) @@ -145,7 +147,7 @@ class DBRuntime(comp.ProgramRuntime): def applications(self): db_type = self.get_option("type") return [ - comp.Program(db_type), + bruntime.Program(db_type), ] def _run_action(self, action, check_exit_code=True): @@ -155,14 +157,14 @@ class DBRuntime(comp.ProgramRuntime): return sh.execute(*cmd, run_as_root=True, check_exit_code=check_exit_code) def start(self): - if self.statii()[0].status != comp.STATUS_STARTED: + if self.statii()[0].status != bruntime.STATUS_STARTED: self._run_action('start') return 1 else: return 0 def stop(self): - if self.statii()[0].status != comp.STATUS_STOPPED: + if self.statii()[0].status != bruntime.STATUS_STOPPED: self._run_action('stop') return 1 else: @@ -176,13 +178,13 @@ class DBRuntime(comp.ProgramRuntime): def statii(self): (sysout, stderr) = self._run_action('status', False) combined = (sysout + stderr).lower() - st = comp.STATUS_UNKNOWN + st = bruntime.STATUS_UNKNOWN if combined.find("running") != -1: - st = comp.STATUS_STARTED + st = bruntime.STATUS_STARTED elif utils.has_any(combined, 'stop', 'unrecognized'): - st = comp.STATUS_STOPPED + st = bruntime.STATUS_STOPPED return [ - comp.ProgramStatus(name=self.applications[0].name, + bruntime.ProgramStatus(name=self.applications[0].name, status=st, details={ 'STDOUT': sysout, diff --git a/anvil/components/glance.py b/anvil/components/glance.py index f97abae9..d4dcc7eb 100644 --- a/anvil/components/glance.py +++ b/anvil/components/glance.py @@ -15,13 +15,16 @@ # under the License. from anvil import colorizer -from anvil import components as comp from anvil import log as logging from anvil import shell as sh from anvil import utils from anvil.utils import OrderedDict +from anvil.components import base_install as binstall +from anvil.components import base_runtime as bruntime +from anvil.components import base_testing as btesting + from anvil.components.helpers import glance as ghelper from anvil.components.helpers import keystone as khelper @@ -39,15 +42,15 @@ SYNC_DB_CMD = [sh.joinpths('$BIN_DIR', 'glance-manage'), BIN_DIR = '/usr/bin/' -class GlanceUninstaller(comp.PythonUninstallComponent): +class GlanceUninstaller(binstall.PythonUninstallComponent): def __init__(self, *args, **kargs): - comp.PythonUninstallComponent.__init__(self, *args, **kargs) + binstall.PythonUninstallComponent.__init__(self, *args, **kargs) self.bin_dir = BIN_DIR -class GlanceInstaller(comp.PythonInstallComponent): +class GlanceInstaller(binstall.PythonInstallComponent): def __init__(self, *args, **kargs): - comp.PythonInstallComponent.__init__(self, *args, **kargs) + binstall.PythonInstallComponent.__init__(self, *args, **kargs) self.bin_dir = BIN_DIR self.configurator = gconf.GlanceConfigurator(self) @@ -60,7 +63,7 @@ class GlanceInstaller(comp.PythonInstallComponent): 'oslo.config')] def post_install(self): - comp.PythonInstallComponent.post_install(self) + binstall.PythonInstallComponent.post_install(self) if self.get_bool_option('db-sync'): self.configurator.setup_db() self._sync_db() @@ -80,14 +83,14 @@ class GlanceInstaller(comp.PythonInstallComponent): def config_params(self, config_fn): # These be used to fill in the configuration params - mp = comp.PythonInstallComponent.config_params(self, config_fn) + mp = binstall.PythonInstallComponent.config_params(self, config_fn) mp['BIN_DIR'] = self.bin_dir return mp -class GlanceRuntime(comp.PythonRuntime): +class GlanceRuntime(bruntime.PythonRuntime): def __init__(self, *args, **kargs): - comp.PythonRuntime.__init__(self, *args, **kargs) + bruntime.PythonRuntime.__init__(self, *args, **kargs) self.bin_dir = BIN_DIR @property @@ -97,7 +100,7 @@ class GlanceRuntime(comp.PythonRuntime): name = "glance-%s" % (name.lower()) path = sh.joinpths(self.bin_dir, name) if sh.is_executable(path): - apps.append(comp.Program(name, path, argv=self._fetch_argv(name))) + apps.append(bruntime.Program(name, path, argv=self._fetch_argv(name))) return apps def _fetch_argv(self, name): @@ -113,7 +116,7 @@ class GlanceRuntime(comp.PythonRuntime): return [u.strip() for u in uris if len(u.strip())] def post_start(self): - comp.PythonRuntime.post_start(self) + bruntime.PythonRuntime.post_start(self) if self.get_bool_option('load-images'): # Install any images that need activating... self.wait_active() @@ -127,3 +130,11 @@ class GlanceRuntime(comp.PythonRuntime): if cache_dir: params['cache_dir'] = cache_dir ghelper.UploadService(**params).install(self._get_image_urls()) + + +class GlanceTester(btesting.PythonTestingComponent): + # NOTE: only run the unit tests + def _get_test_command(self): + base_cmd = btesting.PythonTestingComponent._get_test_command(self) + base_cmd = base_cmd + ['--unittests-only'] + return base_cmd diff --git a/anvil/components/glance_client.py b/anvil/components/glance_client.py index a9e9539d..e624cb03 100644 --- a/anvil/components/glance_client.py +++ b/anvil/components/glance_client.py @@ -14,9 +14,20 @@ # License for the specific language governing permissions and limitations # under the License. -from anvil import components as comp +from anvil.components import base_install as binstall +from anvil.components import base_testing as btesting + +class GlanceClientInstaller(binstall.PythonInstallComponent): + pass -class GlanceClientInstaller(comp.PythonInstallComponent): - def _filter_pip_requires(self, fn, lines): - return [l for l in lines if l.lower().find('keystoneclient') == -1] +class GlanceClientTester(btesting.PythonTestingComponent): + def _use_run_tests(self): + return False + + def _get_test_exclusions(self): + return [ + # These seem to require swift, not always installed... + 'test_ssl_cert_mismatch', + 'test_ssl_cert_subject_alt_name', + ] diff --git a/anvil/components/horizon.py b/anvil/components/horizon.py index f11ebf5d..82864692 100644 --- a/anvil/components/horizon.py +++ b/anvil/components/horizon.py @@ -14,12 +14,14 @@ # License for the specific language governing permissions and limitations # under the License. -from anvil import components as comp from anvil import exceptions as excp from anvil import log as logging from anvil import shell as sh from anvil import utils +from anvil.components import base_install as binstall +from anvil.components import base_runtime as bruntime + from anvil.components.configurators import horizon as hconf import binascii @@ -37,14 +39,14 @@ SECRET_KEY_LEN = 10 BAD_APACHE_USERS = ['root'] -class HorizonUninstaller(comp.PythonUninstallComponent): +class HorizonUninstaller(binstall.PythonUninstallComponent): def __init__(self, *args, **kargs): - comp.PythonUninstallComponent.__init__(self, *args, **kargs) + binstall.PythonUninstallComponent.__init__(self, *args, **kargs) -class HorizonInstaller(comp.PythonInstallComponent): +class HorizonInstaller(binstall.PythonInstallComponent): def __init__(self, *args, **kargs): - comp.PythonInstallComponent.__init__(self, *args, **kargs) + binstall.PythonInstallComponent.__init__(self, *args, **kargs) self.blackhole_dir = sh.joinpths(self.get_option('app_dir'), '.blackhole') self.access_log = sh.joinpths('/var/log/', self.distro.get_command_config('apache', 'name'), @@ -61,7 +63,7 @@ class HorizonInstaller(comp.PythonInstallComponent): return [l for l in lines if not re.search(r'([n|q|s|k|g|c]\w+client)', l, re.I)] def verify(self): - comp.PythonInstallComponent.verify(self) + binstall.PythonInstallComponent.verify(self) self._check_ug() def _check_ug(self): @@ -95,12 +97,12 @@ class HorizonInstaller(comp.PythonInstallComponent): return len(log_fns) def _configure_files(self): - am = comp.PythonInstallComponent._configure_files(self) + am = binstall.PythonInstallComponent._configure_files(self) am += self._setup_logs(self.get_bool_option('clear-logs')) return am def post_install(self): - comp.PythonInstallComponent.post_install(self) + binstall.PythonInstallComponent.post_install(self) if self.get_bool_option('make-blackhole'): self._setup_blackhole() @@ -110,7 +112,7 @@ class HorizonInstaller(comp.PythonInstallComponent): def config_params(self, config_fn): # This dict will be used to fill in the configuration # params with actual values - mp = comp.PythonInstallComponent.config_params(self, config_fn) + mp = binstall.PythonInstallComponent.config_params(self, config_fn) if config_fn == hconf.HORIZON_APACHE_CONF: (user, group) = self._get_apache_user_group() mp['GROUP'] = group @@ -130,9 +132,9 @@ class HorizonInstaller(comp.PythonInstallComponent): return mp -class HorizonRuntime(comp.ProgramRuntime): +class HorizonRuntime(bruntime.ProgramRuntime): def start(self): - if self.statii()[0].status != comp.STATUS_STARTED: + if self.statii()[0].status != bruntime.STATUS_STARTED: self._run_action('start') return 1 else: @@ -149,7 +151,7 @@ class HorizonRuntime(comp.ProgramRuntime): return 1 def stop(self): - if self.statii()[0].status != comp.STATUS_STOPPED: + if self.statii()[0].status != bruntime.STATUS_STOPPED: self._run_action('stop') return 1 else: @@ -158,13 +160,13 @@ class HorizonRuntime(comp.ProgramRuntime): def statii(self): (sysout, stderr) = self._run_action('status', check_exit_code=False) combined = (sysout + stderr).lower() - st = comp.STATUS_UNKNOWN + st = bruntime.STATUS_UNKNOWN if combined.find("is running") != -1: - st = comp.STATUS_STARTED + st = bruntime.STATUS_STARTED elif utils.has_any(combined, 'stopped', 'unrecognized', 'not running'): - st = comp.STATUS_STOPPED + st = bruntime.STATUS_STOPPED return [ - comp.ProgramStatus(name='apache', + bruntime.ProgramStatus(name='apache', status=st, details={ 'STDOUT': sysout, diff --git a/anvil/components/keystone.py b/anvil/components/keystone.py index 517468c4..cce0bdd7 100644 --- a/anvil/components/keystone.py +++ b/anvil/components/keystone.py @@ -15,13 +15,16 @@ # under the License. from anvil import colorizer -from anvil import components as comp from anvil import log as logging from anvil import shell as sh from anvil import utils from anvil.utils import OrderedDict +from anvil.components import base_install as binstall +from anvil.components import base_runtime as bruntime +from anvil.components import base_testing as btesting + from anvil.components.helpers import glance as ghelper from anvil.components.helpers import keystone as khelper from anvil.components.helpers import nova as nhelper @@ -43,14 +46,14 @@ MANAGE_CMD = [sh.joinpths('$BIN_DIR', 'keystone-manage'), '--config-file=$CONFIG_FILE', '--debug', '-v'] -class KeystoneUninstaller(comp.PythonUninstallComponent): +class KeystoneUninstaller(binstall.PythonUninstallComponent): def __init__(self, *args, **kargs): - comp.PythonUninstallComponent.__init__(self, *args, **kargs) + binstall.PythonUninstallComponent.__init__(self, *args, **kargs) -class KeystoneInstaller(comp.PythonInstallComponent): +class KeystoneInstaller(binstall.PythonInstallComponent): def __init__(self, *args, **kargs): - comp.PythonInstallComponent.__init__(self, *args, **kargs) + binstall.PythonInstallComponent.__init__(self, *args, **kargs) self.bin_dir = sh.joinpths(self.get_option('app_dir'), 'bin') self.configurator = kconf.KeystoneConfigurator(self) @@ -64,7 +67,7 @@ class KeystoneInstaller(comp.PythonInstallComponent): 'memcached')] def post_install(self): - comp.PythonInstallComponent.post_install(self) + binstall.PythonInstallComponent.post_install(self) if self.get_bool_option('db-sync'): self.configurator.setup_db() self._sync_db() @@ -107,15 +110,15 @@ class KeystoneInstaller(comp.PythonInstallComponent): def config_params(self, config_fn): # These be used to fill in the configuration params - mp = comp.PythonInstallComponent.config_params(self, config_fn) + mp = binstall.PythonInstallComponent.config_params(self, config_fn) mp['BIN_DIR'] = self.bin_dir mp['CONFIG_FILE'] = sh.joinpths(self.get_option('cfg_dir'), kconf.ROOT_CONF) return mp -class KeystoneRuntime(comp.PythonRuntime): +class KeystoneRuntime(bruntime.PythonRuntime): def __init__(self, *args, **kargs): - comp.PythonRuntime.__init__(self, *args, **kargs) + bruntime.PythonRuntime.__init__(self, *args, **kargs) self.bin_dir = sh.joinpths(self.get_option('app_dir'), 'bin') self.init_fn = sh.joinpths(self.get_option('trace_dir'), INIT_WHAT_HAPPENED) @@ -164,7 +167,7 @@ class KeystoneRuntime(comp.PythonRuntime): name = "keystone-%s" % (name.lower()) path = sh.joinpths(self.bin_dir, name) if sh.is_executable(path): - apps.append(comp.Program(name, path, argv=self._fetch_argv(name))) + apps.append(bruntime.Program(name, path, argv=self._fetch_argv(name))) return apps def _fetch_argv(self, name): @@ -177,9 +180,9 @@ class KeystoneRuntime(comp.PythonRuntime): ] -class KeystoneTester(comp.PythonTestingComponent): +class KeystoneTester(btesting.PythonTestingComponent): # Disable the keystone client integration tests def _get_test_command(self): - base_cmd = comp.PythonTestingComponent._get_test_command(self) + base_cmd = btesting.PythonTestingComponent._get_test_command(self) base_cmd = base_cmd + ['-xintegration'] return base_cmd diff --git a/anvil/components/keystone_client.py b/anvil/components/keystone_client.py index d9c0d3d4..96eed0b8 100644 --- a/anvil/components/keystone_client.py +++ b/anvil/components/keystone_client.py @@ -14,11 +14,11 @@ # License for the specific language governing permissions and limitations # under the License. -from anvil import components as comp from anvil import utils +from anvil.components import base_install as binstall -class KeystoneClientInstaller(comp.PythonInstallComponent): +class KeystoneClientInstaller(binstall.PythonInstallComponent): def _filter_pip_requires(self, fn, lines): return [l for l in lines # Take out entries that aren't really always needed or are diff --git a/anvil/components/nova.py b/anvil/components/nova.py index cf8886c3..71e71035 100644 --- a/anvil/components/nova.py +++ b/anvil/components/nova.py @@ -15,12 +15,14 @@ # under the License. from anvil import colorizer -from anvil import components as comp from anvil import exceptions as excp from anvil import log as logging from anvil import shell as sh from anvil import utils +from anvil.components import base_install as binstall +from anvil.components import base_runtime as bruntime + from anvil.components.configurators import nova as nconf from anvil.components.helpers import nova as nhelper from anvil.components.helpers import rabbit as rhelper @@ -62,9 +64,9 @@ FLOATING_NET_CMDS = [ BIN_DIR = 'bin' -class NovaUninstaller(comp.PythonUninstallComponent): +class NovaUninstaller(binstall.PythonUninstallComponent): def __init__(self, *args, **kargs): - comp.PythonUninstallComponent.__init__(self, *args, **kargs) + binstall.PythonUninstallComponent.__init__(self, *args, **kargs) self.virsh = lv.Virsh(self.get_int_option('service_wait_seconds'), self.distro) def pre_uninstall(self): @@ -90,9 +92,9 @@ class NovaUninstaller(comp.PythonUninstallComponent): LOG.warn("Failed cleaning up nova-compute's dirty laundry due to: %s", e) -class NovaInstaller(comp.PythonInstallComponent): +class NovaInstaller(binstall.PythonInstallComponent): def __init__(self, *args, **kargs): - comp.PythonInstallComponent.__init__(self, *args, **kargs) + binstall.PythonInstallComponent.__init__(self, *args, **kargs) self.configurator = nconf.NovaConfigurator(self) def _filter_pip_requires(self, fn, lines): @@ -114,7 +116,7 @@ class NovaInstaller(comp.PythonInstallComponent): return to_set def verify(self): - comp.PythonInstallComponent.verify(self) + binstall.PythonInstallComponent.verify(self) self.configurator.verify() def warm_configs(self): @@ -141,7 +143,7 @@ class NovaInstaller(comp.PythonInstallComponent): tracewriter=self.tracewriter) def post_install(self): - comp.PythonInstallComponent.post_install(self) + binstall.PythonInstallComponent.post_install(self) # Extra actions to do nova setup if self.get_bool_option('db-sync'): self.configurator.setup_db() @@ -150,15 +152,15 @@ class NovaInstaller(comp.PythonInstallComponent): self._fix_virt() def config_params(self, config_fn): - mp = comp.PythonInstallComponent.config_params(self, config_fn) + mp = binstall.PythonInstallComponent.config_params(self, config_fn) mp['CFG_FILE'] = sh.joinpths(self.get_option('cfg_dir'), nconf.API_CONF) mp['BIN_DIR'] = sh.joinpths(self.get_option('app_dir'), BIN_DIR) return mp -class NovaRuntime(comp.PythonRuntime): +class NovaRuntime(bruntime.PythonRuntime): def __init__(self, *args, **kargs): - comp.PythonRuntime.__init__(self, *args, **kargs) + bruntime.PythonRuntime.__init__(self, *args, **kargs) self.wait_time = self.get_int_option('service_wait_seconds') self.virsh = lv.Virsh(self.wait_time, self.distro) self.config_path = sh.joinpths(self.get_option('cfg_dir'), nconf.API_CONF) @@ -208,12 +210,12 @@ class NovaRuntime(comp.PythonRuntime): name = "nova-%s" % (name.lower()) path = sh.joinpths(self.bin_dir, name) if sh.is_executable(path): - apps.append(comp.Program(name, path, argv=self._fetch_argv(name))) + apps.append(bruntime.Program(name, path, argv=self._fetch_argv(name))) return apps def pre_start(self): # Let the parent class do its thing - comp.PythonRuntime.pre_start(self) + bruntime.PythonRuntime.pre_start(self) virt_driver = utils.canon_virt_driver(self.get_option('virt_driver')) if virt_driver == 'libvirt': virt_type = lv.canon_libvirt_type(self.get_option('libvirt_type')) @@ -229,7 +231,7 @@ class NovaRuntime(comp.PythonRuntime): raise excp.StartException(msg) def app_params(self, program): - params = comp.PythonRuntime.app_params(self, program) + params = bruntime.PythonRuntime.app_params(self, program) params['CFG_FILE'] = self.config_path return params diff --git a/anvil/components/novnc.py b/anvil/components/novnc.py index f2849258..34b8e510 100644 --- a/anvil/components/novnc.py +++ b/anvil/components/novnc.py @@ -14,33 +14,35 @@ # License for the specific language governing permissions and limitations # under the License. -from anvil import components as comp from anvil import shell as sh +from anvil.components import base_install as binstall +from anvil.components import base_runtime as bruntime + # Where the application is really UTIL_DIR = 'utils' VNC_PROXY_APP = 'nova-novncproxy' -class NoVNCUninstaller(comp.PythonUninstallComponent): +class NoVNCUninstaller(binstall.PythonUninstallComponent): pass -class NoVNCInstaller(comp.PythonInstallComponent): +class NoVNCInstaller(binstall.PythonInstallComponent): @property def python_directories(self): # Its python but not one that we need to run setup.py in... return {} -class NoVNCRuntime(comp.PythonRuntime): +class NoVNCRuntime(bruntime.PythonRuntime): @property def applications(self): path = sh.joinpths(self.get_option('app_dir'), UTIL_DIR, VNC_PROXY_APP) argv = ['--config-file', self._get_nova_conf(), '--web', '.'] return [ - comp.Program(VNC_PROXY_APP, path, argv=argv), + bruntime.Program(VNC_PROXY_APP, path, argv=argv), ] def _get_nova_conf(self): diff --git a/anvil/components/openstack_client.py b/anvil/components/openstack_client.py index 62f4f25c..20aec486 100644 --- a/anvil/components/openstack_client.py +++ b/anvil/components/openstack_client.py @@ -14,17 +14,18 @@ # License for the specific language governing permissions and limitations # under the License. -from anvil import components as comp from anvil import utils +from anvil.components import base_install as binstall +from anvil.components import base_testing as btesting -class OpenStackClientInstaller(comp.PythonInstallComponent): +class OpenStackClientInstaller(binstall.PythonInstallComponent): def _filter_pip_requires(self, fn, lines): return [l for l in lines if not utils.has_any(l.lower(), 'keystoneclient', 'novaclient', 'glanceclient')] -class OpenStackClientTester(comp.PythonTestingComponent): +class OpenStackClientTester(btesting.PythonTestingComponent): def _use_run_tests(self): return False diff --git a/anvil/components/pkglist.py b/anvil/components/pkglist.py index 6e4a8d65..96fdb004 100644 --- a/anvil/components/pkglist.py +++ b/anvil/components/pkglist.py @@ -15,10 +15,10 @@ # License for the specific language governing permissions and limitations # under the License. -from anvil import components as comp +from anvil.components import base_install as binstall -class Installer(comp.PythonInstallComponent): +class Installer(binstall.PythonInstallComponent): @property def packages(self): pkg_list = super(Installer, self).packages @@ -38,5 +38,5 @@ class Installer(comp.PythonInstallComponent): return None -class Uninstaller(comp.PythonUninstallComponent): +class Uninstaller(binstall.PythonUninstallComponent): pass diff --git a/anvil/components/quantum.py b/anvil/components/quantum.py index 789cba1d..5648a3ef 100644 --- a/anvil/components/quantum.py +++ b/anvil/components/quantum.py @@ -15,11 +15,13 @@ # under the License. from anvil import colorizer -from anvil import components as comp from anvil import log as logging from anvil import shell as sh from anvil import utils +from anvil.components import base_install as binstall +from anvil.components import base_runtime as bruntime + from anvil.components.configurators import quantum as qconf LOG = logging.getLogger(__name__) @@ -30,13 +32,13 @@ SYNC_DB_CMD = [sh.joinpths("$BIN_DIR", "quantum-db-manage"), BIN_DIR = "bin" -class QuantumUninstaller(comp.PythonUninstallComponent): +class QuantumUninstaller(binstall.PythonUninstallComponent): def __init__(self, *args, **kargs): super(QuantumUninstaller, self).__init__(*args, **kargs) self.bin_dir = sh.joinpths(self.get_option("app_dir"), BIN_DIR) -class QuantumInstaller(comp.PythonInstallComponent): +class QuantumInstaller(binstall.PythonInstallComponent): def __init__(self, *args, **kargs): super(QuantumInstaller, self).__init__(*args, **kargs) self.bin_dir = sh.joinpths(self.get_option("app_dir"), BIN_DIR) @@ -68,7 +70,7 @@ class QuantumInstaller(comp.PythonInstallComponent): return mp -class QuantumRuntime(comp.PythonRuntime): +class QuantumRuntime(bruntime.PythonRuntime): system = "quantum" @@ -87,12 +89,12 @@ class QuantumRuntime(comp.PythonRuntime): name = "%s-%s" % (self.system, name.lower()) path = sh.joinpths(self.bin_dir, name) if sh.is_executable(path): - apps.append(comp.Program( + apps.append(bruntime.Program( name, path, argv=self._fetch_argv(name))) return apps def app_params(self, program): - params = comp.PythonRuntime.app_params(self, program) + params = bruntime.PythonRuntime.app_params(self, program) params["CFG_FILE"] = self.config_path return params diff --git a/anvil/components/rabbit.py b/anvil/components/rabbit.py index 269e503c..0a9b0358 100644 --- a/anvil/components/rabbit.py +++ b/anvil/components/rabbit.py @@ -17,11 +17,13 @@ from tempfile import TemporaryFile from anvil import colorizer -from anvil import components as comp from anvil import log as logging from anvil import shell as sh from anvil import utils +from anvil.components import base_install as binstall +from anvil.components import base_runtime as bruntime + from anvil.components.helpers import rabbit as rhelper LOG = logging.getLogger(__name__) @@ -30,9 +32,9 @@ LOG = logging.getLogger(__name__) RESET_BASE_PW = '' -class RabbitUninstaller(comp.PkgUninstallComponent): +class RabbitUninstaller(binstall.PkgUninstallComponent): def __init__(self, *args, **kargs): - comp.PkgUninstallComponent.__init__(self, *args, **kargs) + binstall.PkgUninstallComponent.__init__(self, *args, **kargs) self.runtime = self.siblings.get('running') def pre_uninstall(self): @@ -50,9 +52,9 @@ class RabbitUninstaller(comp.PkgUninstallComponent): "reset the password to %s before the next install"), colorizer.quote(RESET_BASE_PW)) -class RabbitInstaller(comp.PkgInstallComponent): +class RabbitInstaller(binstall.PkgInstallComponent): def __init__(self, *args, **kargs): - comp.PkgInstallComponent.__init__(self, *args, **kargs) + binstall.PkgInstallComponent.__init__(self, *args, **kargs) self.runtime = self.siblings.get('running') def warm_configs(self): @@ -71,13 +73,13 @@ class RabbitInstaller(comp.PkgInstallComponent): self.runtime.wait_active() def post_install(self): - comp.PkgInstallComponent.post_install(self) + binstall.PkgInstallComponent.post_install(self) self._setup_pw() -class RabbitRuntime(comp.ProgramRuntime): +class RabbitRuntime(bruntime.ProgramRuntime): def start(self): - if self.statii()[0].status != comp.STATUS_STARTED: + if self.statii()[0].status != bruntime.STATUS_STARTED: self._run_action('start') return 1 else: @@ -86,7 +88,7 @@ class RabbitRuntime(comp.ProgramRuntime): @property def applications(self): return [ - comp.Program('rabbit-mq'), + bruntime.Program('rabbit-mq'), ] def statii(self): @@ -94,14 +96,14 @@ class RabbitRuntime(comp.ProgramRuntime): # # I have ever seen (its like a weird mix json+crap) (sysout, stderr) = self._run_action('status', check_exit_code=False) - st = comp.STATUS_UNKNOWN + st = bruntime.STATUS_UNKNOWN combined = (sysout + stderr).lower() if utils.has_any(combined, 'nodedown', "unable to connect to node", 'unrecognized'): - st = comp.STATUS_STOPPED + st = bruntime.STATUS_STOPPED elif combined.find('running_applications') != -1: - st = comp.STATUS_STARTED + st = bruntime.STATUS_STARTED return [ - comp.ProgramStatus(status=st, + bruntime.ProgramStatus(status=st, details={ 'STDOUT': sysout, 'STDERR': stderr, @@ -135,7 +137,7 @@ class RabbitRuntime(comp.ProgramRuntime): return 1 def stop(self): - if self.statii()[0].status != comp.STATUS_STOPPED: + if self.statii()[0].status != bruntime.STATUS_STOPPED: self._run_action('stop') return 1 else: diff --git a/anvil/components/swift_client.py b/anvil/components/swift_client.py index 7532d80f..5ed07ffb 100644 --- a/anvil/components/swift_client.py +++ b/anvil/components/swift_client.py @@ -14,11 +14,12 @@ # License for the specific language governing permissions and limitations # under the License. -from anvil import components as comp from anvil import utils +from anvil.components import base_install as binstall -class SwiftClientInstaller(comp.PythonInstallComponent): + +class SwiftClientInstaller(binstall.PythonInstallComponent): def _filter_pip_requires(self, fn, lines): return [l for l in lines if not utils.has_any(l.lower(), 'keystoneclient')] diff --git a/anvil/packaging/rpm.py b/anvil/packaging/rpm.py index 019a842f..68f6be8c 100644 --- a/anvil/packaging/rpm.py +++ b/anvil/packaging/rpm.py @@ -17,7 +17,6 @@ import copy from anvil import colorizer -from anvil import component as comp from anvil import exceptions as excp from anvil import log as logging from anvil import patcher @@ -26,6 +25,8 @@ from anvil import trace as tr from anvil import type_utils as tu from anvil import utils +from anvil.components import base as comp + from anvil.packaging.helpers import changelog from anvil.packaging.helpers import yum_helper diff --git a/anvil/runners/__init__.py b/anvil/runners/__init__.py index db8cc1de..367c1235 100644 --- a/anvil/runners/__init__.py +++ b/anvil/runners/__init__.py @@ -17,7 +17,7 @@ import abc import weakref -from anvil.components import STATUS_UNKNOWN +from anvil.components.base_runtime import STATUS_UNKNOWN class Runner(object): diff --git a/anvil/runners/fork.py b/anvil/runners/fork.py index f6d28414..a926ea17 100644 --- a/anvil/runners/fork.py +++ b/anvil/runners/fork.py @@ -24,7 +24,7 @@ from anvil import shell as sh from anvil import trace as tr from anvil import utils -from anvil.components import (STATUS_STARTED, STATUS_UNKNOWN) +from anvil.components.base_runtime import (STATUS_STARTED, STATUS_UNKNOWN) LOG = logging.getLogger(__name__) diff --git a/conf/distros/rhel.yaml b/conf/distros/rhel.yaml index a5773b59..8eb43c38 100644 --- a/conf/distros/rhel.yaml +++ b/conf/distros/rhel.yaml @@ -57,26 +57,26 @@ components: install: anvil.components.cinder:CinderInstaller package: anvil.packaging.rpm:PythonPackager running: anvil.components.cinder:CinderRuntime - test: anvil.components:PythonTestingComponent - coverage: anvil.components:PythonTestingComponent + test: anvil.components.base_testing:PythonTestingComponent + coverage: anvil.components.base_testing:PythonTestingComponent uninstall: anvil.components.cinder:CinderUninstaller pips: - name: hp3parclient cinder-client: action_classes: - install: anvil.components:PythonInstallComponent + install: anvil.components.base_install:PythonInstallComponent package: anvil.packaging.rpm:PythonPackager - running: anvil.components:EmptyRuntime - test: anvil.components:PythonTestingComponent - coverage: anvil.components:PythonTestingComponent - uninstall: anvil.components:PythonUninstallComponent + running: anvil.components.base_runtime:EmptyRuntime + test: anvil.components.base_testing:PythonTestingComponent + coverage: anvil.components.base_testing:PythonTestingComponent + uninstall: anvil.components.base_install:PythonUninstallComponent db: action_classes: install: anvil.distros.rhel:DBInstaller package: anvil.packaging.rpm:DependencyPackager running: anvil.components.db:DBRuntime - test: anvil.components:EmptyTestingComponent - coverage: anvil.components:EmptyTestingComponent + coverage: anvil.components.base_testing:EmptyTestingComponent + test: anvil.components.base_testing:EmptyTestingComponent uninstall: anvil.components.db:DBUninstaller packages: - name: mysql @@ -85,9 +85,9 @@ components: action_classes: install: anvil.components.pkglist:Installer package: anvil.packaging.rpm:DependencyPackager - running: anvil.components:EmptyRuntime - test: anvil.components:EmptyTestingComponent - coverage: anvil.components:EmptyTestingComponent + running: anvil.components.base_runtime:EmptyRuntime + test: anvil.components.base_testing:EmptyTestingComponent + coverage: anvil.components.base_testing:EmptyTestingComponent uninstall: anvil.components.pkglist:Uninstaller packages: # Shared system packages @@ -272,10 +272,10 @@ components: action_classes: install: anvil.components.glance_client:GlanceClientInstaller package: anvil.packaging.rpm:PythonPackager - running: anvil.components:EmptyRuntime - coverage: anvil.components:PythonTestingComponent - test: anvil.components:PythonTestingComponent - uninstall: anvil.components:PythonUninstallComponent + running: anvil.components.base_runtime:EmptyRuntime + test: anvil.components.glance_client:GlanceClientTester + coverage: anvil.components.glance_client:GlanceClientTester + uninstall: anvil.components.base_install:PythonUninstallComponent pips: - name: nosexcover - name: setuptools-git @@ -286,8 +286,8 @@ components: install: anvil.distros.rhel:HorizonInstaller package: anvil.packaging.rpm:PythonPackager running: anvil.components.horizon:HorizonRuntime - test: anvil.components:PythonTestingComponent - coverage: anvil.components:PythonTestingComponent + test: anvil.components.base_testing:PythonTestingComponent + coverage: anvil.components.base_testing:PythonTestingComponent uninstall: anvil.components.horizon:HorizonUninstaller pip_to_package: - name: django @@ -320,17 +320,17 @@ components: action_classes: install: anvil.components.keystone_client:KeystoneClientInstaller package: anvil.packaging.rpm:PythonPackager - running: anvil.components:EmptyRuntime - test: anvil.components:PythonTestingComponent - coverage: anvil.components:PythonTestingComponent - uninstall: anvil.components:PythonUninstallComponent + running: anvil.components.base_runtime:EmptyRuntime + test: anvil.components.base_testing:PythonTestingComponent + coverage: anvil.components.base_testing:PythonTestingComponent + uninstall: anvil.components.base_install:PythonUninstallComponent nova: action_classes: install: anvil.distros.rhel:NovaInstaller package: anvil.packaging.rpm:PythonPackager running: anvil.components.nova:NovaRuntime - test: anvil.components:PythonTestingComponent - coverage: anvil.components:PythonTestingComponent + test: anvil.components.base_testing:PythonTestingComponent + coverage: anvil.components.base_testing:PythonTestingComponent uninstall: anvil.components.nova:NovaUninstaller packages: - name: MySQL-python @@ -409,19 +409,19 @@ components: removable: false nova-client: action_classes: - install: anvil.components:PythonInstallComponent + install: anvil.components.base_install:PythonInstallComponent package: anvil.packaging.rpm:PythonPackager - running: anvil.components:EmptyRuntime - test: anvil.components:PythonTestingComponent - coverage: anvil.components:PythonTestingComponent - uninstall: anvil.components:PythonUninstallComponent + running: anvil.components.base_runtime:EmptyRuntime + test: anvil.components.base_testing:PythonTestingComponent + coverage: anvil.components.base_testing:PythonTestingComponent + uninstall: anvil.components.base_install:PythonUninstallComponent no-vnc: action_classes: install: anvil.components.novnc:NoVNCInstaller - package: anvil.components:EmptyPackagingComponent + package: anvil.components.base_install:EmptyPackagingComponent running: anvil.components.novnc:NoVNCRuntime - test: anvil.components:EmptyTestingComponent - coverage: anvil.components:EmptyTestingComponent + test: anvil.components.base_testing:EmptyTestingComponent + coverage: anvil.components.base_testing:EmptyTestingComponent uninstall: anvil.components.novnc:NoVNCUninstaller packages: - name: python-websockify @@ -429,42 +429,42 @@ components: action_classes: install: anvil.components.openstack_client:OpenStackClientInstaller package: anvil.packaging.rpm:PythonPackager - running: anvil.components:EmptyRuntime + running: anvil.components.base_runtime:EmptyRuntime test: anvil.components.openstack_client:OpenStackClientTester coverage: anvil.components.openstack_client:OpenStackClientTester - uninstall: anvil.components:PythonUninstallComponent + uninstall: anvil.components.base_install:PythonUninstallComponent oslo-config: action_classes: - install: anvil.components:PythonInstallComponent + install: anvil.components.base_install:PythonInstallComponent package: anvil.packaging.rpm:PythonPackager - running: anvil.components:EmptyRuntime - test: anvil.components:PythonTestingComponent - coverage: anvil.components:PythonTestingComponent - uninstall: anvil.components:PythonUninstallComponent + running: anvil.components.base_runtime:EmptyRuntime + test: anvil.components.base_testing:PythonTestingComponent + coverage: anvil.components.base_testing:PythonTestingComponent + uninstall: anvil.components.base_install:PythonUninstallComponent oslo-incubator: action_classes: - install: anvil.components:PythonInstallComponent + install: anvil.components.base_install:PythonInstallComponent package: anvil.packaging.rpm:PythonPackager - running: anvil.components:EmptyRuntime - test: anvil.components:PythonTestingComponent - coverage: anvil.components:EmptyTestingComponent - uninstall: anvil.components:PythonUninstallComponent + running: anvil.components.base_runtime:EmptyRuntime + test: anvil.components.base_testing:PythonTestingComponent + coverage: anvil.components.base_testing:PythonTestingComponent + uninstall: anvil.components.base_install:PythonUninstallComponent quantum: action_classes: install: anvil.components.quantum:QuantumInstaller package: anvil.packaging.rpm:PythonPackager running: anvil.components.quantum:QuantumRuntime - test: anvil.components:PythonTestingComponent - coverage: anvil.components:PythonTestingComponent + test: anvil.components.base_testing:PythonTestingComponent + coverage: anvil.components.base_testing:PythonTestingComponent uninstall: anvil.components.quantum:QuantumUninstaller quantum-client: action_classes: - install: anvil.components:PythonInstallComponent + install: anvil.components.base_install:PythonInstallComponent package: anvil.packaging.rpm:PythonPackager - running: anvil.components:EmptyRuntime - test: anvil.components:PythonTestingComponent - coverage: anvil.components:PythonTestingComponent - uninstall: anvil.components:PythonUninstallComponent + running: anvil.components.base_runtime:EmptyRuntime + test: anvil.components.base_testing:PythonTestingComponent + coverage: anvil.components.base_testing:PythonTestingComponent + uninstall: anvil.components.base_install:PythonUninstallComponent pips: - name: cliff-tablib rabbit-mq: @@ -472,8 +472,8 @@ components: install: anvil.components.rabbit:RabbitInstaller package: anvil.packaging.rpm:DependencyPackager running: anvil.distros.rhel:RabbitRuntime - test: anvil.components:EmptyTestingComponent - coverage: anvil.components:EmptyTestingComponent + test: anvil.components.base_testing:EmptyTestingComponent + coverage: anvil.components.base_testing:EmptyTestingComponent uninstall: anvil.components.rabbit:RabbitUninstaller packages: - name: rabbitmq-server @@ -496,8 +496,8 @@ components: action_classes: install: anvil.components.swift_client:SwiftClientInstaller package: anvil.packaging.rpm:PythonPackager - running: anvil.components:EmptyRuntime - test: anvil.components:PythonTestingComponent - coverage: anvil.components:PythonTestingComponent - uninstall: anvil.components:PythonUninstallComponent + running: anvil.components.base_runtime:EmptyRuntime + test: anvil.components.base_testing:PythonTestingComponent + coverage: anvil.components.base_testing:PythonTestingComponent + uninstall: anvil.components.base_install:PythonUninstallComponent ...