diff --git a/README.md b/README.md index cba647e50..01c91a997 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,12 @@ C and C++ compilation Ccache is configured by the base element. Any compilation that honours ccache will be cached. +PyPI +---- + +The pypi element will bind mount a PyPI mirror from the cache dir and configure +pip and easy-install to use it. + Design ====== diff --git a/elements/pypi/README.md b/elements/pypi/README.md new file mode 100644 index 000000000..cb3786f09 --- /dev/null +++ b/elements/pypi/README.md @@ -0,0 +1,40 @@ +Inject a PyPI mirror +==================== + +Bind mounts a PyPI mirror from ~/.cache/image-create/pypi/mirror into the build +environment and temporarily overwrites /root/.pip.conf and .pydistutils.cfg to +use it. + +When online, the master pypi index is supplied as an extra-url, so uncached +dependencies will still be available. When offline, only the mirror is used - a +stale mirror will cause build failures. + +[jeepyb](https://github.com/openstack-infra/jeepyb) can be useful in making a +partial PyPI mirror suitable for building images. For instance: + + * sudo apt-get install libxml2-dev libxslt-dev libmysqlclient-dev libpq-dev \ + libnspr4-dev pkg-config libsqlite3-dev libzmq-dev libffi-dev libldap2-dev \ + libsasl2-dev + + * pip install https://github.com/openstack-infra/jeepyb + + * cat << EOF > mirror.yaml + cache-root: /home/USER/.cache/image-create/pypi/download + + mirrors: + - name: openstack + projects: + - https://github.com/openstack/requirements + output: /home/USER/.cache/image-create/pypi/mirror + + * mkdir -p /home/USER/.cache/image-create/pypi/{download,mirror} + + * run-mirror -b remotes/origin/master --verbose -c mirror.yaml + # This creates and updates the mirror. + +If you have additional packages that are not identified in the global openstack +requirements project, you can include them: + + * pip install -d ~/.cache/image-create/pypi/download/pip/openstack \ + heat-cfntools distribute os-apply-config + run-mirror -b remotes/origin/master --verbose -c mirror.yaml --no-download diff --git a/elements/pypi/extra-data.d/00-mount-pypi-mirror b/elements/pypi/extra-data.d/00-mount-pypi-mirror new file mode 100755 index 000000000..c5c522358 --- /dev/null +++ b/elements/pypi/extra-data.d/00-mount-pypi-mirror @@ -0,0 +1,9 @@ +#!/bin/bash + +set -eu + +MIRROR_SOURCE=~/.cache/image-create/pypi/mirror/ +MIRROR_TARGET=$TMP_MOUNT_PATH/tmp/pypi + +sudo mkdir -p $MIRROR_TARGET +sudo mount --bind $MIRROR_SOURCE $TMP_MOUNT_PATH/tmp/pypi diff --git a/elements/pypi/post-install.d/00-unconfigure-pypi-mirror b/elements/pypi/post-install.d/00-unconfigure-pypi-mirror new file mode 100755 index 000000000..100543831 --- /dev/null +++ b/elements/pypi/post-install.d/00-unconfigure-pypi-mirror @@ -0,0 +1,16 @@ +#!/bin/bash + +set -eu + +if [ -e ~/.pip/pip.conf.orig ]; then + mv ~/.pip/pip.conf{.orig,} +else + rm ~/.pip/pip.conf +fi + + +if [ -e ~/.pydistutils.cfg.orig ]; then + mv ~/.pydistutils.cfg{.orig,} +else + rm ~/.pydistutils.cfg +fi diff --git a/elements/pypi/pre-install.d/00-configure-pypi-mirror b/elements/pypi/pre-install.d/00-configure-pypi-mirror new file mode 100755 index 000000000..402951c51 --- /dev/null +++ b/elements/pypi/pre-install.d/00-configure-pypi-mirror @@ -0,0 +1,32 @@ +#!/bin/bash + +set -eu + +PYPIURL=file:///tmp/pypi +mkdir -p ~/.pip + +if [ -e ~/.pip/pip.conf ]; then + mv ~/.pip/pip.conf{,.orig} +fi + +if [ -e ~/.pydistutils.cfg ]; then + mv ~/.pydistutils.cfg{,.orig} +fi + +if [ -z $DIB_OFFLINE ]; then + ONLINE="extra-index-url = https://pypi.python.org/simple" +else + ONLINE="" +fi + +cat < ~/.pip/pip.conf +[global] +index-url = $PYPIURL +$ONLINE +log = $HOME/pip.log +EOF + +cat < ~/.pydistutils.cfg +[easy_install] +index_url = $PYPIURL +EOF