From c7d3889bd85ba97273c554bc35c9e48fef7371d0 Mon Sep 17 00:00:00 2001 From: Travis Truman Date: Fri, 15 Apr 2016 12:20:47 -0400 Subject: [PATCH] Combine pip_install and pip_lockdown roles Since lockdown is really just a form of specific pip configuration and since the pip_install role already does some pip configuration, it seems logical and desirable for these functions to take place in the same role. This change should pave the way for a simplification of role dependencies and the removal of explicit pip_lockdown role usage with various playbooks that will already have a dependency on pip_install. Change-Id: Ia0fc276c2b501f16d4acf73bbbcad6f80804628e --- .gitignore | 3 + Vagrantfile | 9 + defaults/main.yml | 19 ++ files/pip-link-build.py | 191 ++++++++++++++++++ .../combine_pip_roles-ba524dbaa601e1a1.yaml | 6 + run_tests.sh | 0 tasks/configure.yml | 49 +++++ tasks/install.yml | 58 ++++++ tasks/lockdown.yml | 52 +++++ tasks/main.yml | 75 +------ templates/global.conf.j2 | 11 + templates/link_file.j2 | 1 + 12 files changed, 405 insertions(+), 69 deletions(-) create mode 100644 Vagrantfile create mode 100644 files/pip-link-build.py create mode 100644 releasenotes/notes/combine_pip_roles-ba524dbaa601e1a1.yaml mode change 100644 => 100755 run_tests.sh create mode 100644 tasks/configure.yml create mode 100644 tasks/install.yml create mode 100644 tasks/lockdown.yml create mode 100644 templates/global.conf.j2 create mode 100644 templates/link_file.j2 diff --git a/.gitignore b/.gitignore index 8c6e664..33e009a 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,6 @@ ChangeLog # Files created by releasenotes build releasenotes/build + +# Vagrant testing artifacts +.vagrant diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..cea04e3 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,9 @@ +Vagrant.configure(2) do |config| + config.vm.box = "ubuntu/trusty64" + config.vm.provision "shell", inline: <<-SHELL + sudo su - + cd /vagrant + apt-get update + ./run_tests.sh + SHELL +end \ No newline at end of file diff --git a/defaults/main.yml b/defaults/main.yml index ec7934b..a34462a 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -28,3 +28,22 @@ pip_packages: [] # Validate Certificates when downloading pip. May be set to "no" when proxy server # is intercepting the certificates. pip_validate_certs: "yes" + +pip_lock_to_internal_repo: False + +# Options for pip global +pip_enable_pre_releases: true +pip_timeout: 120 + +# Options for pip install +pip_upgrade: true + +# Drop link files to lock down pip +# Example: +# pip_links: +# - name: "openstack_release" +# link: "{{ openstack_repo_url }}/os-releases/{{ openstack_release }}/" +pip_links: [] + +## Tunable overrides +pip_global_conf_overrides: {} diff --git a/files/pip-link-build.py b/files/pip-link-build.py new file mode 100644 index 0000000..882cee9 --- /dev/null +++ b/files/pip-link-build.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# (c) 2014, Kevin Carter + +""" +This script will build a pip.conf file dynamically based on a simple +configuration layout. The purpose of this script is to allow automation to +deploy parts of the main `pip.conf` file incrementally creating links and +sections as needed. + +Structure: + $HOME/.pip/ + $HOME/.pip/base + $HOME/.pip/links.d + +creates: + $HOME/.pip/pip.conf + + +* The script reads all configuration files from the base directory and then + applies the sections to the main config file at "$HOME/.pip/pip.conf" +* Within the [install] section will be generated with the value `find-links` + built from the link files found in "$HOME/.pip/links.d". +""" + +import ConfigParser +import os +import urlparse + + +def config_files(config_dir_path, extension='.link'): + """Discover all link files. + + :param config_dir_path: ``str`` Path to link directory + :param extension: ``str`` Extension for files + :return: ``list`` + """ + link_files = list() + for root_path, _, pip_files in os.walk(config_dir_path): + for f in pip_files: + if f.endswith(extension): + link_files.append(os.path.join(root_path, f)) + else: + return link_files + + +def pip_links(links_files): + """Read all link files. + + :param links_files: ``list`` List of files to read containing links + :return: `list`` + """ + links = list() + for link in links_files: + with open(link, 'rb') as f: + links.extend(f.readlines()) + else: + return links + + +def load_config(config_file): + """Load config from a file. + + :param config_file: ``str`` path to config file + :return: ``object`` + """ + config = ConfigParser.ConfigParser() + if config_file is None: + return config + + try: + with open(config_file) as f: + config.readfp(f) + except IOError: + return config + else: + return config + + +def set_links(links): + """Set all links and ensure there are no blank lines. + + :param links: ``list`` List of all raw links + :return: ``str`` + """ + pip_find_links = list() + for link in links: + if link != '\n' or not link: + pip_find_links.append(link.rstrip('\n')) + + links = [i for i in list(set(pip_find_links))] + return '\n%s' % '\n'.join(links) + + +def build_main_config(add_conf, main_config): + """Build configuration from all found conf files. + + :param add_conf: ``object`` ConfigParser object + :param main_config: ``object`` ConfigParser object + """ + for section in add_conf.sections(): + try: + main_config.add_section(section) + except ConfigParser.DuplicateSectionError: + pass + + for k, v in add_conf.items(section): + main_config.set(section, k, v) + + +def build_install_section(main_dir_path, main_config): + """Build the install section with links. + + :param main_dir_path: ``str`` Directory path + :param main_config: ``object`` ConfigParser object + """ + links = list() + trusted_host = list() + links_dir = os.path.join(main_dir_path, 'links.d') + if os.path.isdir(links_dir): + _link = config_files(config_dir_path=links_dir, extension='.link') + _links = pip_links(_link) + links.extend(_links) + for _link in _links: + # Make sure that just the hostname/ip is used. + trusted_host.append(urlparse.urlparse(_link).netloc.split(':')[0]) + else: + main_config.set('global', 'trusted-host', set_links(trusted_host)) + + # Add install section if not already found + try: + main_config.add_section('install') + except ConfigParser.DuplicateSectionError: + pass + + # Get all items from the install section + try: + install_items = main_config.items('install') + except ConfigParser.NoSectionError: + install_items = None + + link_strings = set_links(links) + if install_items: + for item in install_items: + if item[0] != 'find-links': + main_config.set('install', *item) + + main_config.set('install', 'find-links', link_strings) + + +def main(user_home=None): + """Run the main application.""" + if not user_home: + user_home = '~/.pip/pip.conf' + main_file_path = os.path.expanduser(user_home) + main_config = load_config(config_file=None) + + main_dir_path = os.path.dirname(main_file_path) + base_dir_path = os.path.join(main_dir_path, 'base') + if os.path.isdir(base_dir_path): + _confs = config_files(base_dir_path, extension='.conf') + for _conf in _confs: + _config = load_config(config_file=_conf) + build_main_config(_config, main_config) + + build_install_section(main_dir_path, main_config) + + # Write out the config file + with open(main_file_path, 'wb') as f: + main_config.write(f) + + +if __name__ == '__main__': + import sys + if len(sys.argv) >= 1: + main(os.path.join(sys.argv[1], '.pip/pip.conf')) + else: + main() diff --git a/releasenotes/notes/combine_pip_roles-ba524dbaa601e1a1.yaml b/releasenotes/notes/combine_pip_roles-ba524dbaa601e1a1.yaml new file mode 100644 index 0000000..4b5e6f7 --- /dev/null +++ b/releasenotes/notes/combine_pip_roles-ba524dbaa601e1a1.yaml @@ -0,0 +1,6 @@ +--- +features: + - The pip_install role can now configure pip to be locked down to the + repository built by OpenStack-Ansible. To enable the lockdown + configuration, deployers may set ``pip_lock_to_internal_repo`` to + ``true`` in ``/etc/openstack_deploy/user_variables.yml``. diff --git a/run_tests.sh b/run_tests.sh old mode 100644 new mode 100755 diff --git a/tasks/configure.yml b/tasks/configure.yml new file mode 100644 index 0000000..92e7416 --- /dev/null +++ b/tasks/configure.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create pip config directory + file: + path: "{{ item }}" + state: "directory" + group: "{{ ansible_user_id }}" + owner: "{{ ansible_user_id }}" + with_items: + - "/opt" + - "{{ ansible_env.HOME }}/.cache" + - "{{ ansible_env.HOME }}/.cache/pip" + - "{{ ansible_env.HOME }}/.pip" + - "{{ ansible_env.HOME }}/.pip/base" + tags: + - pip-directories + +- name: Drop pip file(s) + copy: + src: "selfcheck.json" + dest: "{{ ansible_env.HOME }}/.cache/pip/selfcheck.json" + owner: "{{ ansible_user_id }}" + group: "{{ ansible_user_id }}" + mode: "0644" + tags: + - pip-files + +- name: Drop pip global config(s) + config_template: + src: "global.conf.j2" + dest: "{{ ansible_env.HOME }}/.pip/base/global.conf" + owner: "{{ ansible_user_id }}" + group: "{{ ansible_user_id }}" + mode: "0644" + config_overrides: "{{ pip_global_conf_overrides }}" + config_type: "ini" diff --git a/tasks/install.yml b/tasks/install.yml new file mode 100644 index 0000000..813f8ca --- /dev/null +++ b/tasks/install.yml @@ -0,0 +1,58 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Get Modern PIP + get_url: + url: "{{ pip_upstream_url }}" + dest: "/opt/get-pip.py" + force: "yes" + validate_certs: "{{ pip_validate_certs }}" + register: get_pip + until: get_pip | success + ignore_errors: True + retries: 5 + delay: 2 + tags: + - pip-install-script + +- name: Get Modern PIP using fallback URL + get_url: + url: "{{ pip_fallback_url }}" + dest: "/opt/get-pip.py" + force: "yes" + validate_certs: "{{ pip_validate_certs }}" + when: get_pip | failed + register: get_pip_fallback + until: get_pip_fallback | success + retries: 5 + delay: 2 + tags: + - pip-install-script + +- name: Install PIP + shell: "python /opt/get-pip.py {{ pip_get_pip_options }} {{ pip_packages | map('quote') | join (' ') }}" + ignore_errors: true + register: pip_install + until: pip_install | success + retries: 3 + delay: 2 + +- name: Install PIP (fall back mode) + shell: "python /opt/get-pip.py --isolated {{ pip_get_pip_options }} {{ pip_packages | map('quote') | join (' ') }}" + when: pip_install.rc != 0 + register: pip_install_fall_back + until: pip_install_fall_back | success + retries: 3 + delay: 2 diff --git a/tasks/lockdown.yml b/tasks/lockdown.yml new file mode 100644 index 0000000..f61059e --- /dev/null +++ b/tasks/lockdown.yml @@ -0,0 +1,52 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create pip links directory + file: + path: "{{ item }}" + state: "directory" + group: "{{ ansible_user_id }}" + owner: "{{ ansible_user_id }}" + with_items: + - "{{ ansible_env.HOME }}/.pip/links.d" + tags: + - lock-directories-pip + +- name: Drop pip lockdown file(s) + copy: + src: "pip-link-build.py" + dest: "{{ ansible_env.HOME }}/.pip/pip-link-build.py" + owner: "{{ ansible_user_id }}" + group: "{{ ansible_user_id }}" + mode: "0755" + tags: + - lock-pip-files + +- name: Drop pip link file(s) + template: + src: "link_file.j2" + dest: "{{ ansible_env.HOME }}/.pip/links.d/{{ item.name }}.link" + owner: "{{ ansible_user_id }}" + group: "{{ ansible_user_id }}" + mode: "{{ item.mode|default('0644') }}" + with_items: pip_links + tags: + - lock-pip-files + +- name: Execute pip config builder + command: "{{ ansible_env.HOME }}/.pip/pip-link-build.py {{ ansible_env.HOME }}" + changed_when: false + tags: + - lock-down-pip-conf diff --git a/tasks/main.yml b/tasks/main.yml index f091a84..1af8539 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -13,77 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Create pip config directory - file: - path: "{{ item }}" - state: "directory" - group: "{{ ansible_user_id }}" - owner: "{{ ansible_user_id }}" - with_items: - - "/opt" - - "{{ ansible_env.HOME }}/.cache" - - "{{ ansible_env.HOME }}/.cache/pip" +- include: configure.yml tags: - - pip-directories + - pip-configuration -- name: Drop pip file(s) - copy: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - owner: "{{ ansible_user_id }}" - group: "{{ ansible_user_id }}" - mode: "{{ item.mode|default('0644') }}" - with_items: - - { src: "selfcheck.json", dest: "{{ ansible_env.HOME }}/.cache/pip/selfcheck.json" } - tags: - - pip-files +- include: lockdown.yml + when: + - pip_lock_to_internal_repo | bool -- name: Get Modern PIP - get_url: - url: "{{ pip_upstream_url }}" - dest: "/opt/get-pip.py" - force: "yes" - validate_certs: "{{ pip_validate_certs }}" - register: get_pip - until: get_pip | success - ignore_errors: True - retries: 5 - delay: 2 - tags: - - pip-install-script - - pip-install - -- name: Get Modern PIP using fallback URL - get_url: - url: "{{ pip_fallback_url }}" - dest: "/opt/get-pip.py" - force: "yes" - validate_certs: "{{ pip_validate_certs }}" - when: get_pip | failed - register: get_pip_fallback - until: get_pip_fallback | success - retries: 5 - delay: 2 - tags: - - pip-install-script - - pip-install - -- name: Install PIP - shell: "python /opt/get-pip.py {{ pip_get_pip_options }} {{ pip_packages | map('quote') | join (' ') }}" - ignore_errors: true - register: pip_install - until: pip_install | success - retries: 3 - delay: 2 - tags: - - pip-install - -- name: Install PIP (fall back mode) - shell: "python /opt/get-pip.py --isolated {{ pip_get_pip_options }} {{ pip_packages | map('quote') | join (' ') }}" - when: pip_install.rc != 0 - register: pip_install_fall_back - until: pip_install_fall_back | success - retries: 3 - delay: 2 +- include: install.yml tags: - pip-install diff --git a/templates/global.conf.j2 b/templates/global.conf.j2 new file mode 100644 index 0000000..0441b03 --- /dev/null +++ b/templates/global.conf.j2 @@ -0,0 +1,11 @@ +# {{ ansible_managed }} + +[global] +{% if pip_lock_to_internal_repo %} +no-index = true +{% endif %} +pre = {{ pip_enable_pre_releases }} +timeout = {{ pip_timeout }} + +[install] +upgrade = {{ pip_upgrade }} diff --git a/templates/link_file.j2 b/templates/link_file.j2 new file mode 100644 index 0000000..f711bed --- /dev/null +++ b/templates/link_file.j2 @@ -0,0 +1 @@ +{{ item.link }} \ No newline at end of file