x/anvil
Code Issues Proposed changes

Be smarter about venv dependency installation

Browse Source 
Also be smarter about what we build as venvs
Change-Id: I786dcab353cc28f3f9449ef3d1f8e38dc71f7a93
This commit is contained in:
Joshua Harlow 2015-06-30 14:23:59 -07:00 committed by Kris Lindgren
parent 41136f2edb
commit f1f488db56
4 changed files with 314 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
anvil/packaging/venv.py
View File

@ -35,6 +35,10 @@ from anvil.packaging.helpers import pip_helper
LOG = logging.getLogger(__name__)
def _on_finish(what, time_taken):
LOG.info("%s took %s seconds", what, time_taken)
# TODO(harlowja): think we can remove this...
class VenvInstallHelper(base.InstallHelper):
def pre_install(self, pkg, params=None):
@ -104,6 +108,11 @@ class VenvDependencyHandler(base.DependencyHandler):
files_replaced += 1
return (files_replaced, total_replacements)
def _make_tarball(self, venv_dir, tar_filename):
with contextlib.closing(tarfile.open(tar_filename, "w:gz")) as tfh:
for path in sh.listdir(venv_dir, recursive=True):
tfh.add(path, recursive=False, arcname=path[len(venv_dir):])
def package_finish(self):
super(VenvDependencyHandler, self).package_finish()
for instance in self.instances:
@ -129,9 +138,8 @@ class VenvDependencyHandler(base.DependencyHandler):
tar_filename = sh.joinpths(venv_dir, '%s-venv.tar.gz' % instance.name)
LOG.info("Making tarball of %s built for %s at %s", venv_dir,
instance.name, tar_filename)
with contextlib.closing(tarfile.open(tar_filename, "w:gz")) as tfh:
for path in sh.listdir(venv_dir, recursive=True):
tfh.add(path, recursive=False, arcname=path[len(venv_dir):])
utils.time_it(functools.partial(_on_finish, "Tarball creation"),
self._make_tarball, venv_dir, tar_filename)
def package_start(self):
super(VenvDependencyHandler, self).package_start()
@ -148,33 +156,66 @@ class VenvDependencyHandler(base.DependencyHandler):
self._install_into_venv(instance, self._PREQ_PKGS)
def package_instance(self, instance):
# Skip things that aren't python...
if self._is_buildable(instance):
requires_what = self._filter_download_requires()
requires_keys = set()
for req in requires_what:
if isinstance(req, six.string_types):
req = pip_helper.extract_requirement(req)
requires_keys.add(req.key)
egg_info = getattr(instance, 'egg_info', None)
if egg_info is not None:
# Ensure we have gotten all the things...
test_dependencies = (egg_info.get('test_dependencies', [])
if instance.get_bool_option(
'use_tests_requires', default_value=True)
else [])
for req in itertools.chain(egg_info.get('dependencies', []),
test_dependencies):
if isinstance(req, six.string_types):
req = pip_helper.extract_requirement(req)
if req.key not in requires_keys:
requires_what.append(req)
requires_keys.add(req.key)
self._install_into_venv(instance, requires_what)
self._install_into_venv(instance, [instance.get_option('app_dir')])
else:
if not self._is_buildable(instance):
# Skip things that aren't python...
LOG.warn("Skipping building %s (not python)",
colorizer.quote(instance.name, quote_color='red'))
return
def gather_extras():
extra_reqs = []
for p in instance.get_option("pips", default_value=[]):
req = pip_helper.create_requirement(p['name'], p.get('version'))
extra_reqs.append(req)
if instance.get_bool_option('use_tests_requires', default_value=True):
for p in instance.get_option("test_requires", default_value=[]):
extra_reqs.append(pip_helper.create_requirement(p))
return extra_reqs
all_requires_what = self._filter_download_requires()
all_requires_mapping = {}
for req in all_requires_what:
if isinstance(req, six.string_types):
req = pip_helper.extract_requirement(req)
all_requires_mapping[req.key] = req
direct_requires_what = []
direct_requires_keys = set()
egg_info = getattr(instance, 'egg_info', None)
if egg_info is not None:
# Ensure we have gotten all the things...
test_dependencies = (egg_info.get('test_dependencies', [])
if instance.get_bool_option(
'use_tests_requires', default_value=True)
else [])
for req in itertools.chain(egg_info.get('dependencies', []),
test_dependencies):
if isinstance(req, six.string_types):
req = pip_helper.extract_requirement(req)
if req.key not in direct_requires_keys:
direct_requires_what.append(req)
direct_requires_keys.add(req.key)
requires_what = []
extra_requires_what = gather_extras()
for req in extra_requires_what:
if req.key in all_requires_mapping:
req = all_requires_mapping[req.key]
requires_what.append(req)
try:
direct_requires_keys.remove(req.key)
except KeyError:
pass
for req in direct_requires_what:
if req.key not in direct_requires_keys:
continue
if req.key in all_requires_mapping:
req = all_requires_mapping[req.key]
requires_what.append(req)
utils.time_it(functools.partial(_on_finish, "Dependency installation"),
self._install_into_venv, instance,
requires_what)
utils.time_it(functools.partial(_on_finish, "Instance installation"),
self._install_into_venv, instance,
[instance.get_option('app_dir')])
def download_dependencies(self):
pass

96
conf/origins/juno-venv-2014.2.2.yaml Normal file
View File

@ -0,0 +1,96 @@
platform_overrides:
redhat-7.*|centos-7.*:
forced_pips:
- "sqlalchemy>=0.9.7,<0.9.99"
- "python-ldap>=2.4.6"
- "django-openstack-auth==1.1.7"
- "oslo-config==1.4.0"
- "oslo-incubator==2014.2"
- "oslo-messaging==1.4.1"
- "pycadf==0.6.0"
- "python-cinderclient==1.1.1"
- "python-ceilometerclient==1.0.12"
- "python-glanceclient==0.14.1"
- "python-heatclient==0.2.12"
- "python-ironicclient==0.3.1"
- "python-keystoneclient==0.11.1"
- "python-novaclient==2.20.0"
- "python-neutronclient==2.3.9"
- "python-swiftclient==2.3.1"
- "python-troveclient==1.0.7"
redhat-6.*|centos-6.*:
forced_pips:
- "sqlalchemy>=0.9.7,<0.9.99"
- "python-ldap>=2.4.6"
- "django-openstack-auth==1.1.7"
- "oslo-config==1.4.0"
- "oslo-incubator==2014.2"
- "oslo-messaging==1.4.1"
- "pycadf==0.6.0"
- "python-cinderclient==1.1.1"
- "python-ceilometerclient==1.0.12"
- "python-glanceclient==0.14.1"
- "python-heatclient==0.2.12"
- "python-ironicclient==0.3.1"
- "python-keystoneclient==0.11.1"
- "python-novaclient==2.20.0"
- "python-neutronclient==2.3.9"
- "python-swiftclient==2.3.1"
- "python-troveclient==1.0.7"
ceilometer:
repo: git://github.com/openstack/ceilometer.git
tag: 2014.2.2
use_tests_requires: False
pips:
- name: libvirt-python
- name: functools32
cinder:
repo: git://github.com/openstack/cinder.git
tag: 2014.2.2
pips:
- name: functools32
glance:
repo: git://github.com/openstack/glance.git
tag: 2014.2.2
pips:
- name: functools32
heat:
repo: git://github.com/openstack/heat.git
tag: 2014.2.2
pips:
- name: functools32
horizon:
repo: git://github.com/openstack/horizon.git
tag: 2014.2.2
pips:
- name: lesscpy
- name: functools32
ironic:
repo: git://github.com/openstack/ironic.git
tag: 2014.2.2
pips:
- name: functools32
keystone:
repo: git://github.com/openstack/keystone.git
tag: 2014.2.2
pips:
- name: functools32
nova:
repo: git://github.com/openstack//nova.git
tag: 2014.2.2
pips:
- name: libvirt-python
- name: functools32
neutron:
repo: git://github.com/openstack/neutron.git
tag: 2014.2.2
pips:
- name: pyudev
- name: configobj
- name: functools32
trove:
repo: git://github.com/openstack/trove.git
tag: 2014.2.2
use_tests_requires: False
pips:
- name: functools32

96
conf/origins/kilo-venv-2015.2.1.yaml Normal file
View File

@ -0,0 +1,96 @@
platform_overrides:
redhat-7.*|centos-7.*:
forced_pips:
- "sqlalchemy>=0.9.7,<0.9.99"
- "python-ldap>=2.4.6"
- "django-openstack-auth==1.2.0"
- "oslo-config==1.9.3"
- "oslo-incubator==2015.1.0"
- "oslo-messaging==1.8.2"
- "pycadf==0.6.0"
- "python-cinderclient==1.1.1"
- "python-ceilometerclient==1.0.14"
- "python-glanceclient==0.17.1"
- "python-heatclient==0.4.0"
- "python-ironicclient==0.6.0"
- "python-keystoneclient==1.3.1"
- "python-novaclient==2.23.0"
- "python-neutronclient==2.4.0"
- "python-swiftclient==2.4.0"
- "python-troveclient==1.0.9"
redhat-6.*|centos-6.*:
forced_pips:
- "sqlalchemy>=0.9.7,<0.9.99"
- "python-ldap>=2.4.6"
- "django-openstack-auth==1.2.0"
- "oslo-config==1.9.3"
- "oslo-incubator==2015.1.0"
- "oslo-messaging==1.8.2"
- "pycadf==0.6.0"
- "python-cinderclient==1.1.1"
- "python-ceilometerclient==1.0.14"
- "python-glanceclient==0.17.1"
- "python-heatclient==0.4.0"
- "python-ironicclient==0.6.0"
- "python-keystoneclient==1.3.1"
- "python-novaclient==2.23.0"
- "python-neutronclient==2.4.0"
- "python-swiftclient==2.4.0"
- "python-troveclient==1.0.9"
ceilometer:
repo: git://github.com/openstack/ceilometer.git
tag: 2015.1.0
use_tests_requires: False
pips:
- name: libvirt-python
- name: functools32
cinder:
repo: git://github.com/openstack/cinder.git
tag: 2015.1.0
pips:
- name: functools32
glance:
repo: git://github.com/openstack/glance.git
tag: 2015.1.0
pips:
- name: functools32
heat:
repo: git://github.com/openstack/heat.git
tag: 2015.1.0
pips:
- name: functools32
horizon:
repo: git://github.com/openstack/horizon.git
tag: 2015.1.0
pips:
- name: lesscpy
- name: functools32
ironic:
repo: git://github.com/openstack/ironic.git
tag: 2015.1.0
pips:
- name: functools32
keystone:
repo: git://github.com/openstack/keystone.git
tag: 2015.1.0
pips:
- name: functools32
nova:
repo: git://github.com/openstack//nova.git
tag: 2015.1.0
pips:
- name: libvirt-python
- name: functools32
neutron:
repo: git://github.com/openstack/neutron.git
tag: 2015.1.0
pips:
- name: pyudev
- name: configobj
- name: functools32
trove:
repo: git://github.com/openstack/trove.git
tag: 2015.1.0
use_tests_requires: False
pips:
- name: functools32

53
conf/personas/in-a-box/venv-all.yaml Normal file
View File

@ -0,0 +1,53 @@
---
# Persona that only builds top level components for use with the venv patch
components:
# Core components
- keystone
- glance
- ceilometer
- cinder
- heat
- neutron
- nova
- trove
# Horizon is given a later priority (typically everything is done at the
# same time in stage zero); in its own stage since it requires basically all
# the existing things to be pre-built/started... before it can be...
- horizon
subsystems:
glance:
- api
- registry
keystone:
- all
nova:
- api
- cert
- compute
- conductor
- scheduler
neutron:
- server
- agent
- l3-agent
- metadata-agent
- dhcp-agent
cinder:
- api
- scheduler
- volume
heat:
- api
- api-cfn
- api-cloudwatch
- engine
ceilometer:
- api
- collector
- compute
- central
supports:
- rhel
- fedora
- centos
...