From 38d0beb73ecfb51abd8051b3be65206ba61c75e5 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 7 Apr 2014 15:00:00 -0700 Subject: [PATCH] Add pycadf + fix oslo.messaging requirement Pycadf pulls in an older version of oslo.messaging, as does most of the other packages. Since 1.3.0 is released and we are building it allow the dependency resolver/adjuster to correctly adjust those packages to use the specified version instead of the one found in the requirements files. Change-Id: I59b607839a8b65274a08a4903cafb98ab8900870 --- anvil/decorators.py | 81 +++++++++++++++++++++++++++ anvil/packaging/base.py | 69 ++++++++++++++--------- conf/components/pycadf.yaml | 4 ++ conf/distros/rhel.yaml | 7 +++ conf/origins/havana-2013.2.1.yaml | 2 + conf/origins/havana-2013.2.2.yaml | 2 + conf/origins/havana-2013.2.yaml | 2 + conf/origins/havana-git.yaml | 2 + conf/origins/icehouse.yaml | 3 + conf/origins/master.yaml | 3 + conf/personas/in-a-box/basic-all.yaml | 1 + conf/personas/in-a-box/basic.yaml | 1 + 12 files changed, 151 insertions(+), 26 deletions(-) create mode 100644 anvil/decorators.py create mode 100644 conf/components/pycadf.yaml diff --git a/anvil/decorators.py b/anvil/decorators.py new file mode 100644 index 00000000..f71b5575 --- /dev/null +++ b/anvil/decorators.py @@ -0,0 +1,81 @@ +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import time + +# Various useful decorators... + +# +# Copyright 2011 Christopher Arndt, MIT License +# +# https://wiki.python.org/moin/PythonDecoratorLibrary#Cached_Properties + + +# pylint: disable=C0103 +class cached_property(object): + '''Decorator for read-only properties evaluated only once within TTL period. + + It can be used to created a cached property like this:: + + import random + + # the class containing the property must be a new-style class + class MyClass(object): + # create property whose value is cached for ten minutes + @cached_property(ttl=600) + def randint(self): + # will only be evaluated every 10 min. at maximum. + return random.randint(0, 100) + + The value is cached in the '_cache' attribute of the object instance that + has the property getter method wrapped by this decorator. The '_cache' + attribute value is a dictionary which has a key for every property of the + object which is wrapped by this decorator. Each entry in the cache is + created only when the property is accessed for the first time and is a + two-element tuple with the last computed property value and the last time + it was updated in seconds since the epoch. + + The default time-to-live (TTL) is 300 seconds (5 minutes). Set the TTL to + zero for the cached value to never expire. + + To expire a cached property value manually just do:: + + del instance._cache[] + + ''' + def __init__(self, ttl=300): + self.ttl = ttl + self.fget = None + + def __call__(self, fget, doc=None): + self.fget = fget + self.__doc__ = doc or fget.__doc__ # pylint: disable=W0201 + self.__name__ = fget.__name__ # pylint: disable=W0201 + self.__module__ = fget.__module__ # pylint: disable=W0201 + return self + + def __get__(self, inst, owner): + now = time.time() + try: + value, last_update = inst._cache[self.__name__] + if self.ttl > 0 and now - last_update > self.ttl: + raise AttributeError + except (KeyError, AttributeError): + value = self.fget(inst) + try: + cache = inst._cache + except AttributeError: + cache = inst._cache = {} + cache[self.__name__] = (value, now) + return value diff --git a/anvil/packaging/base.py b/anvil/packaging/base.py index 5e059e3d..3cc6b664 100644 --- a/anvil/packaging/base.py +++ b/anvil/packaging/base.py @@ -21,6 +21,7 @@ import collections from anvil import colorizer +from anvil import decorators from anvil import exceptions as exc from anvil import log as logging from anvil.packaging.helpers import pip_helper @@ -75,8 +76,6 @@ class DependencyHandler(object): self.forced_packages = [] # Instances to there app directory (with a setup.py inside) self.package_dirs = self._get_package_dirs(instances) - # Instantiate this as late as we can. - self._python_names = None # Track what file we create so they can be cleaned up on uninstall. trace_fn = tr.trace_filename(self.root_dir, 'deps') self.tracewriter = tr.TraceWriter(trace_fn, break_if_there=False) @@ -95,17 +94,19 @@ class DependencyHandler(object): else: self.ignore_pips = set(ignore_pips) + @decorators.cached_property(ttl=0) + def _python_eggs(self): + egg_infos = [] + for i in self.instances: + try: + egg_infos.append(dict(i.egg_info)) + except AttributeError: + pass + return egg_infos + @property def python_names(self): - if self._python_names is None: - names = [] - for i in self.instances: - try: - names.append(i.egg_info['name']) - except AttributeError: - pass - self._python_names = names - return self._python_names + return [e['name'] for e in self._python_eggs] @staticmethod def _get_package_dirs(instances): @@ -117,19 +118,24 @@ class DependencyHandler(object): return package_dirs def package_start(self): + create_requirement = pip_helper.create_requirement + + def gather_extras(instance): + pips = [] + for p in instance.get_option("pips", default_value=[]): + req = create_requirement(p['name'], p.get('version')) + pips.append(str(req)) + requires_files = list(getattr(instance, 'requires_files', [])) + if instance.get_bool_option('use_tests_requires', default_value=True): + requires_files.extend(getattr(instance, 'test_requires_files', [])) + return (pips, requires_files) + requires_files = [] extra_pips = [] for i in self.instances: - requires_files.extend(getattr(i, 'requires_files', ())) - if i.get_bool_option('use_tests_requires', default_value=True): - requires_files.extend(getattr(i, 'test_requires_files', ())) - - # Ensure we include any extra pips that are desired. - i_extra_pips = i.get_option("pips") or [] - for i_pip in i_extra_pips: - extra_req = pip_helper.create_requirement(i_pip['name'], - i_pip.get('version')) - extra_pips.append(str(extra_req)) + instance_pips, instance_requires_files = gather_extras(i) + extra_pips.extend(instance_pips) + requires_files.extend(instance_requires_files) requires_files = filter(sh.isfile, requires_files) self._gather_pips_to_install(requires_files, sorted(set(extra_pips))) self._clean_pip_requires(requires_files) @@ -192,6 +198,10 @@ class DependencyHandler(object): Updates `self.forced_packages` and `self.pips_to_install`. Writes requirements to `self.gathered_requires_filename`. """ + + def sort_req(r1, r2): + return cmp(r1.key, r2.key) + extra_pips = extra_pips or [] cmdline = [self.multipip_executable] cmdline = cmdline + extra_pips + ["-r"] + requires_files @@ -205,7 +215,9 @@ class DependencyHandler(object): stdout, stderr = sh.execute(cmdline, check_exit_code=False) self.pips_to_install = list(utils.splitlines_not_empty(stdout)) sh.write_file(self.gathered_requires_filename, "\n".join(self.pips_to_install)) - utils.log_iterable(sorted(self.pips_to_install), logger=LOG, + pips_to_install = pip_helper.read_requirement_files([self.gathered_requires_filename]) + pips_to_install = sorted(pips_to_install, cmp=sort_req) + utils.log_iterable(pips_to_install, logger=LOG, header="Full known python dependency list") incompatibles = collections.defaultdict(list) @@ -234,12 +246,17 @@ class DependencyHandler(object): # Translate those that we altered requirements for into a set of forced # requirements file (and associated list). - self.forced_packages = [] + self.forced_packages = [e['req'] for e in self._python_eggs] + forced_packages_keys = [e.key for e in self.forced_packages] for req in [pip_helper.extract_requirement(line) for line in self.pips_to_install]: - if req.key in incompatibles: + if req.key in incompatibles and req.key not in forced_packages_keys: self.forced_packages.append(req) - sh.write_file(self.forced_requires_filename, - "\n".join([str(req) for req in self.forced_packages])) + forced_packages_keys.append(req.key) + self.forced_packages = sorted(self.forced_packages, cmp=sort_req) + forced_packages = [str(req) for req in self.forced_packages] + utils.log_iterable(forced_packages, logger=LOG, + header="Forced python dependencies") + sh.write_file(self.forced_requires_filename, "\n".join(forced_packages)) def _filter_download_requires(self): """Shrinks the pips that were downloaded into a smaller set. diff --git a/conf/components/pycadf.yaml b/conf/components/pycadf.yaml new file mode 100644 index 00000000..45840968 --- /dev/null +++ b/conf/components/pycadf.yaml @@ -0,0 +1,4 @@ +# Settings for component pycadf +--- + +... diff --git a/conf/distros/rhel.yaml b/conf/distros/rhel.yaml index aa4e7576..1bd73c1b 100644 --- a/conf/distros/rhel.yaml +++ b/conf/distros/rhel.yaml @@ -301,6 +301,13 @@ components: test: anvil.components.base_testing:PythonTestingComponent coverage: anvil.components.base_testing:PythonTestingComponent uninstall: anvil.components.base_install:PkgUninstallComponent + pycadf: + action_classes: + install: anvil.components.base_install:PythonInstallComponent + running: anvil.components.base_runtime:EmptyRuntime + test: anvil.components.base_testing:PythonTestingComponent + coverage: anvil.components.base_testing:PythonTestingComponent + uninstall: anvil.components.base_install:PkgUninstallComponent oslo-messaging: action_classes: install: anvil.components.base_install:PythonInstallComponent diff --git a/conf/origins/havana-2013.2.1.yaml b/conf/origins/havana-2013.2.1.yaml index 0ae2fd44..a671163c 100644 --- a/conf/origins/havana-2013.2.1.yaml +++ b/conf/origins/havana-2013.2.1.yaml @@ -69,3 +69,5 @@ trove: tag: 2013.2.1 oslo-messaging: disabled: True +pycadf: + disabled: True diff --git a/conf/origins/havana-2013.2.2.yaml b/conf/origins/havana-2013.2.2.yaml index 6c82dc4c..b387bc5e 100644 --- a/conf/origins/havana-2013.2.2.yaml +++ b/conf/origins/havana-2013.2.2.yaml @@ -69,3 +69,5 @@ trove: tag: 2013.2.2 oslo-messaging: disabled: True +pycadf: + disabled: True diff --git a/conf/origins/havana-2013.2.yaml b/conf/origins/havana-2013.2.yaml index a9eb9068..5ea1e576 100644 --- a/conf/origins/havana-2013.2.yaml +++ b/conf/origins/havana-2013.2.yaml @@ -69,3 +69,5 @@ trove: tag: 2013.2 oslo-messaging: disabled: True +pycadf: + disabled: True diff --git a/conf/origins/havana-git.yaml b/conf/origins/havana-git.yaml index 201c66aa..612abbec 100644 --- a/conf/origins/havana-git.yaml +++ b/conf/origins/havana-git.yaml @@ -72,3 +72,5 @@ trove: branch: stable/havana oslo-messaging: disabled: True +pycadf: + disabled: True diff --git a/conf/origins/icehouse.yaml b/conf/origins/icehouse.yaml index dda26366..77264bc4 100644 --- a/conf/origins/icehouse.yaml +++ b/conf/origins/icehouse.yaml @@ -70,3 +70,6 @@ trove: oslo-messaging: repo: git://github.com/openstack/oslo.messaging.git tag: 1.3.0 +pycadf: + repo: git://github.com/openstack/pycadf.git + tag: 0.5 diff --git a/conf/origins/master.yaml b/conf/origins/master.yaml index 9d984782..cee04033 100644 --- a/conf/origins/master.yaml +++ b/conf/origins/master.yaml @@ -73,3 +73,6 @@ trove-client: trove: repo: git://github.com/openstack/trove.git branch: master +pycadf: + repo: git://github.com/openstack/pycadf.git + tag: master diff --git a/conf/personas/in-a-box/basic-all.yaml b/conf/personas/in-a-box/basic-all.yaml index f28c39b7..9f619ba4 100644 --- a/conf/personas/in-a-box/basic-all.yaml +++ b/conf/personas/in-a-box/basic-all.yaml @@ -7,6 +7,7 @@ components: - rabbit-mq - oslo-config - oslo-messaging +- pycadf - keystone - keystone-client - glance diff --git a/conf/personas/in-a-box/basic.yaml b/conf/personas/in-a-box/basic.yaml index 37bc00f2..ea4d4ca4 100644 --- a/conf/personas/in-a-box/basic.yaml +++ b/conf/personas/in-a-box/basic.yaml @@ -6,6 +6,7 @@ components: - rabbit-mq - oslo-config - oslo-messaging +- pycadf - keystone # Client used by many components - keystone-client