Combine pip_install and pip_lockdown roles
Since lockdown is really just a form of specific pip configuration and since the pip_install role already does some pip configuration, it seems logical and desirable for these functions to take place in the same role. This change should pave the way for a simplification of role dependencies and the removal of explicit pip_lockdown role usage with various playbooks that will already have a dependency on pip_install. Change-Id: Ia0fc276c2b501f16d4acf73bbbcad6f80804628e
This commit is contained in:
parent
681bca546b
commit
c7d3889bd8
|
@ -57,3 +57,6 @@ ChangeLog
|
|||
|
||||
# Files created by releasenotes build
|
||||
releasenotes/build
|
||||
|
||||
# Vagrant testing artifacts
|
||||
.vagrant
|
||||
|
|
|
@ -0,0 +1,9 @@
|
|||
Vagrant.configure(2) do |config|
|
||||
config.vm.box = "ubuntu/trusty64"
|
||||
config.vm.provision "shell", inline: <<-SHELL
|
||||
sudo su -
|
||||
cd /vagrant
|
||||
apt-get update
|
||||
./run_tests.sh
|
||||
SHELL
|
||||
end
|
|
@ -28,3 +28,22 @@ pip_packages: []
|
|||
# Validate Certificates when downloading pip. May be set to "no" when proxy server
|
||||
# is intercepting the certificates.
|
||||
pip_validate_certs: "yes"
|
||||
|
||||
pip_lock_to_internal_repo: False
|
||||
|
||||
# Options for pip global
|
||||
pip_enable_pre_releases: true
|
||||
pip_timeout: 120
|
||||
|
||||
# Options for pip install
|
||||
pip_upgrade: true
|
||||
|
||||
# Drop link files to lock down pip
|
||||
# Example:
|
||||
# pip_links:
|
||||
# - name: "openstack_release"
|
||||
# link: "{{ openstack_repo_url }}/os-releases/{{ openstack_release }}/"
|
||||
pip_links: []
|
||||
|
||||
## Tunable overrides
|
||||
pip_global_conf_overrides: {}
|
||||
|
|
|
@ -0,0 +1,191 @@
|
|||
#!/usr/bin/env python
|
||||
# Copyright 2014, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
# (c) 2014, Kevin Carter <kevin.carter@rackspace.com>
|
||||
|
||||
"""
|
||||
This script will build a pip.conf file dynamically based on a simple
|
||||
configuration layout. The purpose of this script is to allow automation to
|
||||
deploy parts of the main `pip.conf` file incrementally creating links and
|
||||
sections as needed.
|
||||
|
||||
Structure:
|
||||
$HOME/.pip/
|
||||
$HOME/.pip/base
|
||||
$HOME/.pip/links.d
|
||||
|
||||
creates:
|
||||
$HOME/.pip/pip.conf
|
||||
|
||||
|
||||
* The script reads all configuration files from the base directory and then
|
||||
applies the sections to the main config file at "$HOME/.pip/pip.conf"
|
||||
* Within the [install] section will be generated with the value `find-links`
|
||||
built from the link files found in "$HOME/.pip/links.d".
|
||||
"""
|
||||
|
||||
import ConfigParser
|
||||
import os
|
||||
import urlparse
|
||||
|
||||
|
||||
def config_files(config_dir_path, extension='.link'):
|
||||
"""Discover all link files.
|
||||
|
||||
:param config_dir_path: ``str`` Path to link directory
|
||||
:param extension: ``str`` Extension for files
|
||||
:return: ``list``
|
||||
"""
|
||||
link_files = list()
|
||||
for root_path, _, pip_files in os.walk(config_dir_path):
|
||||
for f in pip_files:
|
||||
if f.endswith(extension):
|
||||
link_files.append(os.path.join(root_path, f))
|
||||
else:
|
||||
return link_files
|
||||
|
||||
|
||||
def pip_links(links_files):
|
||||
"""Read all link files.
|
||||
|
||||
:param links_files: ``list`` List of files to read containing links
|
||||
:return: `list``
|
||||
"""
|
||||
links = list()
|
||||
for link in links_files:
|
||||
with open(link, 'rb') as f:
|
||||
links.extend(f.readlines())
|
||||
else:
|
||||
return links
|
||||
|
||||
|
||||
def load_config(config_file):
|
||||
"""Load config from a file.
|
||||
|
||||
:param config_file: ``str`` path to config file
|
||||
:return: ``object``
|
||||
"""
|
||||
config = ConfigParser.ConfigParser()
|
||||
if config_file is None:
|
||||
return config
|
||||
|
||||
try:
|
||||
with open(config_file) as f:
|
||||
config.readfp(f)
|
||||
except IOError:
|
||||
return config
|
||||
else:
|
||||
return config
|
||||
|
||||
|
||||
def set_links(links):
|
||||
"""Set all links and ensure there are no blank lines.
|
||||
|
||||
:param links: ``list`` List of all raw links
|
||||
:return: ``str``
|
||||
"""
|
||||
pip_find_links = list()
|
||||
for link in links:
|
||||
if link != '\n' or not link:
|
||||
pip_find_links.append(link.rstrip('\n'))
|
||||
|
||||
links = [i for i in list(set(pip_find_links))]
|
||||
return '\n%s' % '\n'.join(links)
|
||||
|
||||
|
||||
def build_main_config(add_conf, main_config):
|
||||
"""Build configuration from all found conf files.
|
||||
|
||||
:param add_conf: ``object`` ConfigParser object
|
||||
:param main_config: ``object`` ConfigParser object
|
||||
"""
|
||||
for section in add_conf.sections():
|
||||
try:
|
||||
main_config.add_section(section)
|
||||
except ConfigParser.DuplicateSectionError:
|
||||
pass
|
||||
|
||||
for k, v in add_conf.items(section):
|
||||
main_config.set(section, k, v)
|
||||
|
||||
|
||||
def build_install_section(main_dir_path, main_config):
|
||||
"""Build the install section with links.
|
||||
|
||||
:param main_dir_path: ``str`` Directory path
|
||||
:param main_config: ``object`` ConfigParser object
|
||||
"""
|
||||
links = list()
|
||||
trusted_host = list()
|
||||
links_dir = os.path.join(main_dir_path, 'links.d')
|
||||
if os.path.isdir(links_dir):
|
||||
_link = config_files(config_dir_path=links_dir, extension='.link')
|
||||
_links = pip_links(_link)
|
||||
links.extend(_links)
|
||||
for _link in _links:
|
||||
# Make sure that just the hostname/ip is used.
|
||||
trusted_host.append(urlparse.urlparse(_link).netloc.split(':')[0])
|
||||
else:
|
||||
main_config.set('global', 'trusted-host', set_links(trusted_host))
|
||||
|
||||
# Add install section if not already found
|
||||
try:
|
||||
main_config.add_section('install')
|
||||
except ConfigParser.DuplicateSectionError:
|
||||
pass
|
||||
|
||||
# Get all items from the install section
|
||||
try:
|
||||
install_items = main_config.items('install')
|
||||
except ConfigParser.NoSectionError:
|
||||
install_items = None
|
||||
|
||||
link_strings = set_links(links)
|
||||
if install_items:
|
||||
for item in install_items:
|
||||
if item[0] != 'find-links':
|
||||
main_config.set('install', *item)
|
||||
|
||||
main_config.set('install', 'find-links', link_strings)
|
||||
|
||||
|
||||
def main(user_home=None):
|
||||
"""Run the main application."""
|
||||
if not user_home:
|
||||
user_home = '~/.pip/pip.conf'
|
||||
main_file_path = os.path.expanduser(user_home)
|
||||
main_config = load_config(config_file=None)
|
||||
|
||||
main_dir_path = os.path.dirname(main_file_path)
|
||||
base_dir_path = os.path.join(main_dir_path, 'base')
|
||||
if os.path.isdir(base_dir_path):
|
||||
_confs = config_files(base_dir_path, extension='.conf')
|
||||
for _conf in _confs:
|
||||
_config = load_config(config_file=_conf)
|
||||
build_main_config(_config, main_config)
|
||||
|
||||
build_install_section(main_dir_path, main_config)
|
||||
|
||||
# Write out the config file
|
||||
with open(main_file_path, 'wb') as f:
|
||||
main_config.write(f)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
if len(sys.argv) >= 1:
|
||||
main(os.path.join(sys.argv[1], '.pip/pip.conf'))
|
||||
else:
|
||||
main()
|
|
@ -0,0 +1,6 @@
|
|||
---
|
||||
features:
|
||||
- The pip_install role can now configure pip to be locked down to the
|
||||
repository built by OpenStack-Ansible. To enable the lockdown
|
||||
configuration, deployers may set ``pip_lock_to_internal_repo`` to
|
||||
``true`` in ``/etc/openstack_deploy/user_variables.yml``.
|
|
@ -0,0 +1,49 @@
|
|||
---
|
||||
# Copyright 2014, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- name: Create pip config directory
|
||||
file:
|
||||
path: "{{ item }}"
|
||||
state: "directory"
|
||||
group: "{{ ansible_user_id }}"
|
||||
owner: "{{ ansible_user_id }}"
|
||||
with_items:
|
||||
- "/opt"
|
||||
- "{{ ansible_env.HOME }}/.cache"
|
||||
- "{{ ansible_env.HOME }}/.cache/pip"
|
||||
- "{{ ansible_env.HOME }}/.pip"
|
||||
- "{{ ansible_env.HOME }}/.pip/base"
|
||||
tags:
|
||||
- pip-directories
|
||||
|
||||
- name: Drop pip file(s)
|
||||
copy:
|
||||
src: "selfcheck.json"
|
||||
dest: "{{ ansible_env.HOME }}/.cache/pip/selfcheck.json"
|
||||
owner: "{{ ansible_user_id }}"
|
||||
group: "{{ ansible_user_id }}"
|
||||
mode: "0644"
|
||||
tags:
|
||||
- pip-files
|
||||
|
||||
- name: Drop pip global config(s)
|
||||
config_template:
|
||||
src: "global.conf.j2"
|
||||
dest: "{{ ansible_env.HOME }}/.pip/base/global.conf"
|
||||
owner: "{{ ansible_user_id }}"
|
||||
group: "{{ ansible_user_id }}"
|
||||
mode: "0644"
|
||||
config_overrides: "{{ pip_global_conf_overrides }}"
|
||||
config_type: "ini"
|
|
@ -0,0 +1,58 @@
|
|||
---
|
||||
# Copyright 2014, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- name: Get Modern PIP
|
||||
get_url:
|
||||
url: "{{ pip_upstream_url }}"
|
||||
dest: "/opt/get-pip.py"
|
||||
force: "yes"
|
||||
validate_certs: "{{ pip_validate_certs }}"
|
||||
register: get_pip
|
||||
until: get_pip | success
|
||||
ignore_errors: True
|
||||
retries: 5
|
||||
delay: 2
|
||||
tags:
|
||||
- pip-install-script
|
||||
|
||||
- name: Get Modern PIP using fallback URL
|
||||
get_url:
|
||||
url: "{{ pip_fallback_url }}"
|
||||
dest: "/opt/get-pip.py"
|
||||
force: "yes"
|
||||
validate_certs: "{{ pip_validate_certs }}"
|
||||
when: get_pip | failed
|
||||
register: get_pip_fallback
|
||||
until: get_pip_fallback | success
|
||||
retries: 5
|
||||
delay: 2
|
||||
tags:
|
||||
- pip-install-script
|
||||
|
||||
- name: Install PIP
|
||||
shell: "python /opt/get-pip.py {{ pip_get_pip_options }} {{ pip_packages | map('quote') | join (' ') }}"
|
||||
ignore_errors: true
|
||||
register: pip_install
|
||||
until: pip_install | success
|
||||
retries: 3
|
||||
delay: 2
|
||||
|
||||
- name: Install PIP (fall back mode)
|
||||
shell: "python /opt/get-pip.py --isolated {{ pip_get_pip_options }} {{ pip_packages | map('quote') | join (' ') }}"
|
||||
when: pip_install.rc != 0
|
||||
register: pip_install_fall_back
|
||||
until: pip_install_fall_back | success
|
||||
retries: 3
|
||||
delay: 2
|
|
@ -0,0 +1,52 @@
|
|||
---
|
||||
# Copyright 2014, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- name: Create pip links directory
|
||||
file:
|
||||
path: "{{ item }}"
|
||||
state: "directory"
|
||||
group: "{{ ansible_user_id }}"
|
||||
owner: "{{ ansible_user_id }}"
|
||||
with_items:
|
||||
- "{{ ansible_env.HOME }}/.pip/links.d"
|
||||
tags:
|
||||
- lock-directories-pip
|
||||
|
||||
- name: Drop pip lockdown file(s)
|
||||
copy:
|
||||
src: "pip-link-build.py"
|
||||
dest: "{{ ansible_env.HOME }}/.pip/pip-link-build.py"
|
||||
owner: "{{ ansible_user_id }}"
|
||||
group: "{{ ansible_user_id }}"
|
||||
mode: "0755"
|
||||
tags:
|
||||
- lock-pip-files
|
||||
|
||||
- name: Drop pip link file(s)
|
||||
template:
|
||||
src: "link_file.j2"
|
||||
dest: "{{ ansible_env.HOME }}/.pip/links.d/{{ item.name }}.link"
|
||||
owner: "{{ ansible_user_id }}"
|
||||
group: "{{ ansible_user_id }}"
|
||||
mode: "{{ item.mode|default('0644') }}"
|
||||
with_items: pip_links
|
||||
tags:
|
||||
- lock-pip-files
|
||||
|
||||
- name: Execute pip config builder
|
||||
command: "{{ ansible_env.HOME }}/.pip/pip-link-build.py {{ ansible_env.HOME }}"
|
||||
changed_when: false
|
||||
tags:
|
||||
- lock-down-pip-conf
|
|
@ -13,77 +13,14 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- name: Create pip config directory
|
||||
file:
|
||||
path: "{{ item }}"
|
||||
state: "directory"
|
||||
group: "{{ ansible_user_id }}"
|
||||
owner: "{{ ansible_user_id }}"
|
||||
with_items:
|
||||
- "/opt"
|
||||
- "{{ ansible_env.HOME }}/.cache"
|
||||
- "{{ ansible_env.HOME }}/.cache/pip"
|
||||
- include: configure.yml
|
||||
tags:
|
||||
- pip-directories
|
||||
- pip-configuration
|
||||
|
||||
- name: Drop pip file(s)
|
||||
copy:
|
||||
src: "{{ item.src }}"
|
||||
dest: "{{ item.dest }}"
|
||||
owner: "{{ ansible_user_id }}"
|
||||
group: "{{ ansible_user_id }}"
|
||||
mode: "{{ item.mode|default('0644') }}"
|
||||
with_items:
|
||||
- { src: "selfcheck.json", dest: "{{ ansible_env.HOME }}/.cache/pip/selfcheck.json" }
|
||||
tags:
|
||||
- pip-files
|
||||
- include: lockdown.yml
|
||||
when:
|
||||
- pip_lock_to_internal_repo | bool
|
||||
|
||||
- name: Get Modern PIP
|
||||
get_url:
|
||||
url: "{{ pip_upstream_url }}"
|
||||
dest: "/opt/get-pip.py"
|
||||
force: "yes"
|
||||
validate_certs: "{{ pip_validate_certs }}"
|
||||
register: get_pip
|
||||
until: get_pip | success
|
||||
ignore_errors: True
|
||||
retries: 5
|
||||
delay: 2
|
||||
tags:
|
||||
- pip-install-script
|
||||
- pip-install
|
||||
|
||||
- name: Get Modern PIP using fallback URL
|
||||
get_url:
|
||||
url: "{{ pip_fallback_url }}"
|
||||
dest: "/opt/get-pip.py"
|
||||
force: "yes"
|
||||
validate_certs: "{{ pip_validate_certs }}"
|
||||
when: get_pip | failed
|
||||
register: get_pip_fallback
|
||||
until: get_pip_fallback | success
|
||||
retries: 5
|
||||
delay: 2
|
||||
tags:
|
||||
- pip-install-script
|
||||
- pip-install
|
||||
|
||||
- name: Install PIP
|
||||
shell: "python /opt/get-pip.py {{ pip_get_pip_options }} {{ pip_packages | map('quote') | join (' ') }}"
|
||||
ignore_errors: true
|
||||
register: pip_install
|
||||
until: pip_install | success
|
||||
retries: 3
|
||||
delay: 2
|
||||
tags:
|
||||
- pip-install
|
||||
|
||||
- name: Install PIP (fall back mode)
|
||||
shell: "python /opt/get-pip.py --isolated {{ pip_get_pip_options }} {{ pip_packages | map('quote') | join (' ') }}"
|
||||
when: pip_install.rc != 0
|
||||
register: pip_install_fall_back
|
||||
until: pip_install_fall_back | success
|
||||
retries: 3
|
||||
delay: 2
|
||||
- include: install.yml
|
||||
tags:
|
||||
- pip-install
|
||||
|
|
|
@ -0,0 +1,11 @@
|
|||
# {{ ansible_managed }}
|
||||
|
||||
[global]
|
||||
{% if pip_lock_to_internal_repo %}
|
||||
no-index = true
|
||||
{% endif %}
|
||||
pre = {{ pip_enable_pre_releases }}
|
||||
timeout = {{ pip_timeout }}
|
||||
|
||||
[install]
|
||||
upgrade = {{ pip_upgrade }}
|
|
@ -0,0 +1 @@
|
|||
{{ item.link }}
|
Loading…
Reference in New Issue