From 8dc6d50b1678894fea0d79d26daee51d09729ab6 Mon Sep 17 00:00:00 2001 From: Oliver Walsh Date: Tue, 13 Aug 2019 18:28:53 +0100 Subject: [PATCH] Add role/playbook to manage nova image cache Adds a tripleo-nova-image-cache role to manage the nova local image cache on remote compute nodes. For multi-site/stack deployments the image can be downloaded once and distributed to the remaining nodes in the stack to minimise WAN traffic. Also adds a playbook to run the role using the single/multi-stack inventory and the overcloudrc environment variables. Change-Id: Ib5aaa22f6cf307181d8f34cf89f9f24619b43004 Implements: blueprint tripleo-nova-cache-mgmt --- .../modules-tripleo_nova_image_cache.rst | 14 + .../roles/role-tripleo-nova-image-cache.rst | 6 + .../action/tripleo_nova_image_cache.py | 178 +++++++++++ .../modules/tripleo_nova_image_cache.py | 283 ++++++++++++++++++ .../playbooks/tripleo_nova_image_cache.yml | 61 ++++ .../defaults/main.yml | 27 ++ .../tripleo-nova-image-cache/meta/main.yml | 44 +++ .../molecule/default/Dockerfile | 37 +++ .../molecule/default/molecule.yml | 48 +++ .../molecule/default/playbook.yml | 21 ++ .../molecule/default/prepare.yml | 21 ++ .../tripleo-nova-image-cache/tasks/cache.yml | 44 +++ .../tripleo-nova-image-cache/tasks/main.yml | 57 ++++ .../tasks/uncache.yml | 21 ++ .../tripleo-nova-image-cache/vars/main.yml | 25 ++ zuul.d/molecule.yaml | 11 + 16 files changed, 898 insertions(+) create mode 100644 doc/source/modules/modules-tripleo_nova_image_cache.rst create mode 100644 doc/source/roles/role-tripleo-nova-image-cache.rst create mode 100644 tripleo_ansible/ansible_plugins/action/tripleo_nova_image_cache.py create mode 100644 tripleo_ansible/ansible_plugins/modules/tripleo_nova_image_cache.py create mode 100644 tripleo_ansible/playbooks/tripleo_nova_image_cache.yml create mode 100644 tripleo_ansible/roles/tripleo-nova-image-cache/defaults/main.yml create mode 100644 tripleo_ansible/roles/tripleo-nova-image-cache/meta/main.yml create mode 100644 tripleo_ansible/roles/tripleo-nova-image-cache/molecule/default/Dockerfile create mode 100644 tripleo_ansible/roles/tripleo-nova-image-cache/molecule/default/molecule.yml create mode 100644 tripleo_ansible/roles/tripleo-nova-image-cache/molecule/default/playbook.yml create mode 100644 tripleo_ansible/roles/tripleo-nova-image-cache/molecule/default/prepare.yml create mode 100644 tripleo_ansible/roles/tripleo-nova-image-cache/tasks/cache.yml create mode 100644 tripleo_ansible/roles/tripleo-nova-image-cache/tasks/main.yml create mode 100644 tripleo_ansible/roles/tripleo-nova-image-cache/tasks/uncache.yml create mode 100644 tripleo_ansible/roles/tripleo-nova-image-cache/vars/main.yml diff --git a/doc/source/modules/modules-tripleo_nova_image_cache.rst b/doc/source/modules/modules-tripleo_nova_image_cache.rst new file mode 100644 index 000000000..2d8d66ff0 --- /dev/null +++ b/doc/source/modules/modules-tripleo_nova_image_cache.rst @@ -0,0 +1,14 @@ +================================= +Module - tripleo_nova_image_cache +================================= + + +This module provides for the following ansible plugin: + + * tripleo_nova_image_cache + + +.. ansibleautoplugin:: + :module: tripleo_ansible/ansible_plugins/modules/tripleo_nova_image_cache.py + :documentation: true + :examples: true diff --git a/doc/source/roles/role-tripleo-nova-image-cache.rst b/doc/source/roles/role-tripleo-nova-image-cache.rst new file mode 100644 index 000000000..597d57b44 --- /dev/null +++ b/doc/source/roles/role-tripleo-nova-image-cache.rst @@ -0,0 +1,6 @@ +=============================== +Role - tripleo-nova-image-cache +=============================== + +.. ansibleautoplugin:: + :role: tripleo_ansible/roles/tripleo-nova-image-cache diff --git a/tripleo_ansible/ansible_plugins/action/tripleo_nova_image_cache.py b/tripleo_ansible/ansible_plugins/action/tripleo_nova_image_cache.py new file mode 100644 index 000000000..bc887e7c9 --- /dev/null +++ b/tripleo_ansible/ansible_plugins/action/tripleo_nova_image_cache.py @@ -0,0 +1,178 @@ +#!/usr/bin/python +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import +__metaclass__ = type + +import hashlib +import os +import uuid + + +from ansible.errors import AnsibleAction +from ansible.errors import AnsibleActionFail +from ansible.errors import AnsibleError +from ansible.plugins.action import ActionBase +from ansible.plugins.action import display + + +class ActionModule(ActionBase): + TRANSFERS_FILES = False + + def run(self, tmp=None, task_vars=None): + if task_vars is None: + task_vars = dict() + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + scp_source = self._task.args.get('scp_source', None) + scp_continue = self._task.args.get('scp_continue_on_error', False) + state = self._task.args.get('state', 'present') + + try: + # Ensure it's a valid uuid + image_id = str(uuid.UUID(self._task.args.get('id'))) + except ValueError: + raise AnsibleError( + "Invalid image id: {}".format( + self._task.args.get('id') + ) + ) + + cache_dir = task_vars.get( + 'tripleo_nova_cache_dir', + '/var/lib/nova/instances/_base' + ) + cache_fn = hashlib.sha1(image_id.encode('utf-8')).hexdigest() + cache_file = os.path.join(cache_dir, cache_fn) + cache_tmp = os.path.join( + cache_dir, + 'ansible_tripleo_nova_cache_tmp_{}'.format(os.getpid()) + ) + tmp_file = os.path.join(cache_tmp, cache_fn) + container_cli = task_vars.get('container_cli', 'podman') + + result.update({'actions': []}) + + try: + # Ensure target directory exists + command_args = { + '_raw_params': + ( + "{} exec -u nova nova_compute /bin/bash -c " + "\"mkdir -p '{}'; chmod 755 '{}'\"" + ).format(container_cli, cache_dir, cache_dir), + 'creates': cache_dir + + } + command_task_vars = {'become': True} + command_result = self._execute_module( + 'command', + module_args=command_args, + task_vars=command_task_vars + ) + command_result['name'] = 'Ensure nova cache dir exists' + result['actions'].append(command_result) + cmd = self._connection._shell.exists(cache_file) + cache_file_exists_res = self._low_level_execute_command( + cmd, + sudoable=True + ) + cache_file_exists = self._parse_returned_data( + cache_file_exists_res).get('rc', 0) == 0 + result['actions'].append({ + 'name': 'Check if cache file exists', + 'exists': cache_file_exists + }) + + new_module_args = self._task.args.copy() + new_module_args.pop('scp_source', None) + new_module_args['_cache_dir'] = cache_dir + new_module_args['_cache_file'] = cache_file + + if state == 'present' and \ + not cache_file_exists and \ + scp_source is not None: + # Create tmp dir + command_args = { + '_raw_params': + ( + "{} exec -u nova nova_compute /bin/bash -c " + "\"mkdir -p '{}'; chmod 755 '{}'\"" + ).format(container_cli, cache_tmp, cache_tmp), + } + command_task_vars = {'become': True} + command_result = self._execute_module( + 'command', + module_args=command_args, + task_vars=command_task_vars + ) + command_result['name'] = 'Create tmp dir' + result['actions'].append(command_result) + + command_args = { + '_raw_params': + "{} exec -u nova nova_compute scp {}:'{}' '{}'".format( + container_cli, + scp_source, + cache_file, + cache_tmp + ) + } + command_task_vars = {'become': True, 'ignore_errors': True} + command_result = self._execute_module( + 'command', + module_args=command_args, + task_vars=command_task_vars) + command_result['name'] = 'Fetch image from {}'.format( + scp_source + ) + result['actions'].append(command_result) + if command_result['rc'] == 0: + new_module_args['_prefetched_path'] = tmp_file + elif not scp_continue: + raise AnsibleActionFail( + '{} failed: {}'.format( + command_result['name'], + command_result['msg'] + ) + ) + + command_result = self._execute_module( + 'tripleo_nova_image_cache', + module_args=new_module_args, + task_vars=task_vars + ) + result['actions'] += command_result.pop('actions', []) + result.update(command_result) + + except AnsibleAction as e: + result.update(e.result) + finally: + cmd = self._connection._shell.remove(cache_tmp, recurse=True) + tmp_rm_res = self._low_level_execute_command(cmd, sudoable=True) + tmp_rm_data = self._parse_returned_data(tmp_rm_res) + if tmp_rm_data.get('rc', 0) != 0: + display.warning( + 'Error deleting remote temporary files ' + ' (rc: %s, stderr: %s})' % ( + tmp_rm_res.get('rc'), + tmp_rm_res.get('stderr', 'No error string available.') + ) + ) + self._remove_tmp_path(self._connection._shell.tmpdir) + return result diff --git a/tripleo_ansible/ansible_plugins/modules/tripleo_nova_image_cache.py b/tripleo_ansible/ansible_plugins/modules/tripleo_nova_image_cache.py new file mode 100644 index 000000000..86e60346f --- /dev/null +++ b/tripleo_ansible/ansible_plugins/modules/tripleo_nova_image_cache.py @@ -0,0 +1,283 @@ +#!/usr/bin/python +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import +__metaclass__ = type + +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.openstack import openstack_cloud_from_module +from ansible.module_utils.openstack import openstack_full_argument_spec +from ansible.module_utils.openstack import openstack_module_kwargs +import datetime +import hashlib +import os +import tempfile +import time + +ANSIBLE_METADATA = { + 'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community' +} + + +DOCUMENTATION = ''' +--- +module: tripleo_nova_image_cache +short_description: Manage Nova image cache on TripleO OpenStack deployment +version_added: "2.0" +author: "Oliver Walsh (@owalsh)" +description: + - Manage Nova image cache on TripleO OpenStack deployment +options: + id: + description: + - ID of the image to cache + required: true + ttl: + description: + - Ensure the image remains cached for at least ttl days + default: 7 + state: + description: + - Whether the image be present in cache or expired + (nova should delete it later if it is no longer used) + choices: [present, expired] + default: present + scp_source: + description: + - Attempt to scp the image from this nova-compute host + scp_continue_on_error: + description: + - Fallback to image download if scp fails + default: false + +requirements: ["openstacksdk"] +''' + +EXAMPLES = ''' +- name: Cache image for at least 2 weeks + tripleo_nova_image_cache: + id: ec151bd1-aab4-413c-b577-ced089e7d3f8 + ttl: 14 + +- name: Allow nova to cleanup this image from cache + tripleo_nova_image_cache: + id: ec151bd1-aab4-413c-b577-ced089e7d3f8 + state: expired + +- name: Cache image, try to copy from existing host + tripleo_nova_image_cache: + id: ec151bd1-aab4-413c-b577-ced089e7d3f8 + ttl: 14 + scp_source: nova-compute-0 + scp_continue_on_error: true + +''' + + +def ttl_to_ts(ttl_days): + """Return a timestamp at midnight UTC ttl_days in the future""" + epoch = datetime.datetime.utcfromtimestamp(0) + midnight_today = datetime.datetime.utcnow().replace( + hour=0, minute=0, second=0, microsecond=0) + expiry = midnight_today + datetime.timedelta(days=ttl_days+1) + return (expiry - epoch).total_seconds() + + +def main(): + + argument_spec = openstack_full_argument_spec( + id=dict(required=True), + ttl=dict(default=7, type='int', min=1), + state=dict(default='present', choices=['expired', 'present']), + _cache_dir=dict(required=True), + _cache_file=dict(required=True), + _chunk_size=dict(default=64 * 1024, type='int'), + _prefetched_path=dict(default=None), + scp_continue_on_error=dict(default=False, type='bool') + ) + module_kwargs = openstack_module_kwargs() + module = AnsibleModule(argument_spec, **module_kwargs) + + image_id = module.params['id'] + cache_dir = module.params['_cache_dir'] + cache_file = module.params['_cache_file'] + chunk_size = module.params['_chunk_size'] + prefetched_path = module.params['_prefetched_path'] + scp_continue = module.params['scp_continue_on_error'] + + ttl_days = module.params['ttl'] + + result = dict( + changed=False, + actions=[], + image=None, + cache_file='', + exists_in_cache=False, + mtime=0 + ) + + sdk, cloud = openstack_cloud_from_module(module, min_version='0.11.3') + + try: + result['exists_in_cache'] = exists_in_cache = os.path.exists( + cache_file) + if exists_in_cache: + result['cache_file'] = cache_file + + image = cloud.image.find_image(name_or_id=image_id) + exists_in_glance = image is not None + if exists_in_glance: + result['image'] = image.to_dict() + + if module.params['state'] == 'present': + if not exists_in_cache: + + if not exists_in_glance: + module.fail_json( + msg="Image not found in glance: %s" % image_id) + + md5 = hashlib.md5() + if prefetched_path: + result['actions'].append({ + 'name': 'Verify pre-fetched image checksum' + }) + with open(prefetched_path, 'rb') as prefetched_image_file: + while True: + chunk = prefetched_image_file.read(chunk_size) + if not chunk: + break + md5.update(chunk) + prefetched_checksum = md5.hexdigest() + if prefetched_checksum == image.checksum: + result['actions'].append({ + 'name': 'Verify pre-fetched image', + 'result': True, + 'expected_md5': image.checksum, + 'actual_md5': prefetched_checksum + }) + # FIXME: chown to the container nova uid (42436) + # until we can run within the container + os.chown(prefetched_path, 42436, 42436) + os.rename(prefetched_path, cache_file) + else: + result['actions'].append({ + 'name': 'Verify pre-fetched image', + 'result': False, + 'expected_md5': image.checksum, + 'actual_md5': prefetched_checksum + }) + if not scp_continue: + module.fail_json( + msg="Pre-fetched image checksum failed") + # Ignore it and download direct from glance. + # As we did not create it we should not remove it. + prefetched_path = '' + + if not prefetched_path: + with tempfile.NamedTemporaryFile( + 'wb', + dir=cache_dir, + delete=False) as temp_cache_file: + try: + md5 = hashlib.md5() + image_stream = cloud.image.download_image( + image, + stream=True + ) + try: + for chunk in image_stream.iter_content( + chunk_size=chunk_size): + md5.update(chunk) + temp_cache_file.write(chunk) + finally: + image_stream.close() + temp_cache_file.close() + + download_checksum = md5.hexdigest() + if download_checksum != image.checksum: + result['actions'].append({ + 'name': 'Verify downloaded image', + 'result': False, + 'expected_md5': image.checksum, + 'actual_md5': download_checksum + }) + module.fail_json( + msg="Image data does not match checksum") + result['actions'].append({ + 'name': 'Verify downloaded image', + 'result': True, + 'expected_md5': image.checksum, + 'actual_md5': download_checksum + }) + + # FIXME: chown to the container nova uid (42436) + # until we can run within the container + os.chown(temp_cache_file.name, 42436, 42436) + os.rename(temp_cache_file.name, cache_file) + result['changed'] = True + finally: + try: + os.unlink(temp_cache_file.name) + except Exception: + pass + + # Set the mtime in the future to prevent nova cleanup + cache_file_stat = os.stat(cache_file) + expiry_ts = ttl_to_ts(ttl_days) + now = time.time() + if cache_file_stat.st_mtime != expiry_ts: + os.utime(cache_file, (now, expiry_ts)) + result['actions'].append({ + 'name': 'Update mtime', + 'from': cache_file_stat.st_mtime, + 'to': expiry_ts + }) + result['changed'] = True + + else: # expired + if not exists_in_cache: + result['changed'] = False + else: + # Set the mtime to epoch to enable nova cleanup + now = time.time() + ts = 0 + cache_file_stat = os.stat(cache_file) + if cache_file_stat.st_mtime > ts: + os.utime(cache_file, (now, ts)) + result['actions'].append({ + 'name': 'Update mtime', + 'from': cache_file_stat.st_mtime, + 'to': ts + }) + result['changed'] = True + + cache_file_stat = os.stat(cache_file) + result['mtime'] = cache_file_stat.st_mtime + result['expires'] = time.strftime( + "%a, %d %b %Y %H:%M:%S %z", + time.localtime(cache_file_stat.st_mtime) + ) + + module.exit_json(**result) + + except sdk.exceptions.OpenStackCloudException as e: + module.fail_json(msg=str(e), extra_data=e.extra_data) + + +if __name__ == "__main__": + main() diff --git a/tripleo_ansible/playbooks/tripleo_nova_image_cache.yml b/tripleo_ansible/playbooks/tripleo_nova_image_cache.yml new file mode 100644 index 000000000..ef960db0a --- /dev/null +++ b/tripleo_ansible/playbooks/tripleo_nova_image_cache.yml @@ -0,0 +1,61 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: TripleO Nova image cache management + hosts: "{{ tripleo_nova_image_cache_plan + '_nova_compute' if tripleo_nova_image_cache_plan|default('') else 'nova_compute'}}" + environment: + # export everything in overcloudrc for openstacksdk + NOVA_VERSION: "{{ lookup('env', 'NOVA_VERSION') }}" + COMPUTE_API_VERSION: "{{ lookup('env', 'COMPUTE_API_VERSION') }}" + OS_USERNAME: "{{ lookup('env', 'OS_USERNAME') }}" + OS_PROJECT_NAME: "{{ lookup('env', 'OS_PROJECT_NAME') }}" + OS_USER_DOMAIN_NAME: "{{ lookup('env', 'OS_USER_DOMAIN_NAME') }}" + OS_PROJECT_DOMAIN_NAME: "{{ lookup('env', 'OS_PROJECT_DOMAIN_NAME') }}" + OS_NO_CACHE: "{{ lookup('env', 'OS_NO_CACHE') }}" + no_proxy: "{{ lookup('env', 'no_proxy') }}" + OS_AUTH_TYPE: "{{ lookup('env', 'OS_AUTH_TYPE') }}" + OS_PASSWORD: "{{ lookup('env', 'OS_PASSWORD') }}" + OS_AUTH_URL: "{{ lookup('env', 'OS_AUTH_URL') }}" + OS_IDENTITY_API_VERSION: "{{ lookup('env', 'OS_IDENTITY_API_VERSION') }}" + OS_IMAGE_API_VERSION: "{{ lookup('env', 'OS_IMAGE_API_VERSION') }}" + OS_VOLUME_API_VERSION: "{{ lookup('env', 'OS_VOLUME_API_VERSION') }}" + OS_REGION_NAME: "{{ lookup('env', 'OS_REGION_NAME') }}" + roles: + - role: tripleo-nova-image-cache + +# Standard inventory: +# +# ansible-playbook -i inventory.yml --extra-vars "@test_args1.yml" tripleo_nova_image_cache.yml +# test_args1.yml +# tripleo_nova_image_cache_images: +# - id: d23c6b8f-e166-4a02-afd8-0ae8d6f73f18 +# state: expired +# - id: 81bbb16-d589-4730-be70-822a82ab6bb9 +# ttl: 28 +# +# Multi-stack inventory: +# +# ansible-playbook -i inventory_multi.yml --extra-vars "@test_args2.yml" tripleo_nova_image_cache.yml +# test_args2.yml: +# tripleo_nova_image_cache_plan: edge0 +# tripleo_nova_image_cache_images: +# - id: d23c6b8f-e166-4a02-afd8-0ae8d6f73f18 +# state: expired +# - id: 81bbb16-d589-4730-be70-822a82ab6bb9 +# ttl: 28 +# tripleo_nova_image_cache_use_proxy: true +# tripleo_nova_image_cache_proxy_hostname: compute-1 # optional, first nova_compute host is used otherwise +# diff --git a/tripleo_ansible/roles/tripleo-nova-image-cache/defaults/main.yml b/tripleo_ansible/roles/tripleo-nova-image-cache/defaults/main.yml new file mode 100644 index 000000000..1eccddb8c --- /dev/null +++ b/tripleo_ansible/roles/tripleo-nova-image-cache/defaults/main.yml @@ -0,0 +1,27 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +# All variables intended for modification should place placed in this file. + +# All variables within this role should have a prefix of "tripleo_nova_image_cache" +tripleo_nova_image_cache_debug: false +tripleo_nova_image_cache_images: [] +tripleo_nova_image_cache_use_proxy: false +tripleo_nova_image_cache_proxy_hostname: "{{ ansible_play_batch[0] }}" +tripleo_nova_image_cache_proxy_network: internal_api +tripleo_nova_image_cache_ignore_proxy_error: false +tripleo_nova_image_cache_proxy_concurrency: 1 diff --git a/tripleo_ansible/roles/tripleo-nova-image-cache/meta/main.yml b/tripleo_ansible/roles/tripleo-nova-image-cache/meta/main.yml new file mode 100644 index 000000000..0289d0548 --- /dev/null +++ b/tripleo_ansible/roles/tripleo-nova-image-cache/meta/main.yml @@ -0,0 +1,44 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +galaxy_info: + author: OpenStack + description: TripleO OpenStack Role -- tripleo-nova-image-cache + company: Red Hat + license: Apache-2.0 + min_ansible_version: 2.7 + # + # Provide a list of supported platforms, and for each platform a list of versions. + # If you don't wish to enumerate all versions for a particular platform, use 'all'. + # To view available platforms and versions (or releases), visit: + # https://galaxy.ansible.com/api/v1/platforms/ + # + platforms: + - name: Fedora + versions: + - 28 + - name: CentOS + versions: + - 7 + + galaxy_tags: + - tripleo + + +# List your role dependencies here, one per line. Be sure to remove the '[]' above, +# if you add dependencies to this list. +dependencies: [] diff --git a/tripleo_ansible/roles/tripleo-nova-image-cache/molecule/default/Dockerfile b/tripleo_ansible/roles/tripleo-nova-image-cache/molecule/default/Dockerfile new file mode 100644 index 000000000..c58ca6538 --- /dev/null +++ b/tripleo_ansible/roles/tripleo-nova-image-cache/molecule/default/Dockerfile @@ -0,0 +1,37 @@ +# Molecule managed +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install sudo python*-devel python*-dnf bash {{ item.pkg_extras | default('') }} && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl python-setuptools bash {{ item.pkg_extras | default('') }} && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml {{ item.pkg_extras | default('') }} && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates {{ item.pkg_extras | default('') }}; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates {{ item.pkg_extras | default('') }} && xbps-remove -O; fi + +{% for pkg in item.easy_install | default([]) %} +# install pip for centos where there is no python-pip rpm in default repos +RUN easy_install {{ pkg }} +{% endfor %} + + +CMD ["sh", "-c", "while true; do sleep 10000; done"] \ No newline at end of file diff --git a/tripleo_ansible/roles/tripleo-nova-image-cache/molecule/default/molecule.yml b/tripleo_ansible/roles/tripleo-nova-image-cache/molecule/default/molecule.yml new file mode 100644 index 000000000..a92a446f3 --- /dev/null +++ b/tripleo_ansible/roles/tripleo-nova-image-cache/molecule/default/molecule.yml @@ -0,0 +1,48 @@ +--- +driver: + name: docker + +log: true + +platforms: + - name: centos7 + hostname: centos7 + image: centos:7 + dockerfile: Dockerfile + pkg_extras: python-setuptools + easy_install: + - pip + environment: &env + http_proxy: "{{ lookup('env', 'http_proxy') }}" + https_proxy: "{{ lookup('env', 'https_proxy') }}" + + - name: centos8 + hostname: centos8 + image: centos:8 + dockerfile: Dockerfile + pkg_extras: python*-setuptools + environment: + <<: *env + +provisioner: + name: ansible + log: true + env: + ANSIBLE_STDOUT_CALLBACK: yaml + +scenario: + test_sequence: + - destroy + - create + - prepare + - converge + - verify + - destroy + +lint: + enabled: false + +verifier: + name: testinfra + lint: + name: flake8 diff --git a/tripleo_ansible/roles/tripleo-nova-image-cache/molecule/default/playbook.yml b/tripleo_ansible/roles/tripleo-nova-image-cache/molecule/default/playbook.yml new file mode 100644 index 000000000..8b78478a8 --- /dev/null +++ b/tripleo_ansible/roles/tripleo-nova-image-cache/molecule/default/playbook.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +- name: Converge + hosts: all + roles: + - role: "tripleo-nova-image-cache" diff --git a/tripleo_ansible/roles/tripleo-nova-image-cache/molecule/default/prepare.yml b/tripleo_ansible/roles/tripleo-nova-image-cache/molecule/default/prepare.yml new file mode 100644 index 000000000..ef85c3128 --- /dev/null +++ b/tripleo_ansible/roles/tripleo-nova-image-cache/molecule/default/prepare.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +- name: Prepare + hosts: all + roles: + - role: test_deps diff --git a/tripleo_ansible/roles/tripleo-nova-image-cache/tasks/cache.yml b/tripleo_ansible/roles/tripleo-nova-image-cache/tasks/cache.yml new file mode 100644 index 000000000..7e1267532 --- /dev/null +++ b/tripleo_ansible/roles/tripleo-nova-image-cache/tasks/cache.yml @@ -0,0 +1,44 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Show proxy host + debug: + msg: "Proxy host is {{ tripleo_nova_image_cache_proxy_hostname }}" + run_once: true + when: + - tripleo_nova_image_cache_use_proxy | bool + +- name: "Cache image {{ image.id }}" + become: true + tripleo_nova_image_cache: + id: "{{ image.id }}" + ttl: "{{ image.ttl|default(omit) }}" + state: "{{ image.state|default(omit) }}" + any_errors_fatal: "{{ true if tripleo_nova_image_cache_use_proxy and tripleo_nova_image_cache_is_proxy_host else false }}" + when: + - not (tripleo_nova_image_cache_use_proxy | bool) or (tripleo_nova_image_cache_is_proxy_host | bool) + +- name: "Cache image (via proxy) {{ image.id }}" + become: true + tripleo_nova_image_cache: + id: "{{ image.id }}" + ttl: "{{ image.ttl|default(omit) }}" + state: "{{ image.state|default(omit) }}" + scp_source: "{{ tripleo_nova_image_cache_proxy_source_ip }}" + scp_continue_on_error: "{{ tripleo_nova_image_cache_ignore_proxy_error }}" + when: + - tripleo_nova_image_cache_use_proxy | bool + - not (tripleo_nova_image_cache_is_proxy_host | bool) diff --git a/tripleo_ansible/roles/tripleo-nova-image-cache/tasks/main.yml b/tripleo_ansible/roles/tripleo-nova-image-cache/tasks/main.yml new file mode 100644 index 000000000..3da9f7daf --- /dev/null +++ b/tripleo_ansible/roles/tripleo-nova-image-cache/tasks/main.yml @@ -0,0 +1,57 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +# "tripleo-nova-image-cache" will search for and load any operating system variable file + +# found within the "vars/" path. If no OS files are found the task will skip. +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - skip: true + files: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Check proxy host + fail: + msg: >- + Inventory does not include the `tripleo_nova_image_cache_proxy_hostname` host ({{ tripleo_nova_image_cache_proxy_hostname }}) + when: + - tripleo_nova_image_cache_use_proxy | bool + - tripleo_nova_image_cache_proxy_hostname not in hostvars + +- name: Cache images + include_tasks: cache.yml + vars: + image: "{{ item }}" + loop: "{{ [ + tripleo_nova_image_cache_images|selectattr('state', 'undefined')|list, + tripleo_nova_image_cache_images|selectattr('state', 'defined')|rejectattr('state', 'equalto', 'expired')|list + ]|flatten }}" + + +- name: Uncache images + include_tasks: uncache.yml + vars: + image: "{{ item }}" + loop: "{{ tripleo_nova_image_cache_images|rejectattr('state', 'undefined')|selectattr('state', 'equalto', 'expired')|list }}" diff --git a/tripleo_ansible/roles/tripleo-nova-image-cache/tasks/uncache.yml b/tripleo_ansible/roles/tripleo-nova-image-cache/tasks/uncache.yml new file mode 100644 index 000000000..c2fec15d8 --- /dev/null +++ b/tripleo_ansible/roles/tripleo-nova-image-cache/tasks/uncache.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: "Uncache image {{ image.id }}" + become: true + tripleo_nova_image_cache: + id: "{{ image.id }}" + state: expired diff --git a/tripleo_ansible/roles/tripleo-nova-image-cache/vars/main.yml b/tripleo_ansible/roles/tripleo-nova-image-cache/vars/main.yml new file mode 100644 index 000000000..0c294b297 --- /dev/null +++ b/tripleo_ansible/roles/tripleo-nova-image-cache/vars/main.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +# While options found within the vars/ path can be overridden using extra +# vars, items within this path are considered part of the role and not +# intended to be modified. + +# All variables within this role should have a prefix of "tripleo_nova_image_cache" +# +tripleo_nova_image_cache_proxy_source_ip: "{{ hostvars[tripleo_nova_image_cache_proxy_hostname][tripleo_nova_image_cache_proxy_network ~ '_ip'] }}" +tripleo_nova_image_cache_is_proxy_host: "{{ tripleo_nova_image_cache_proxy_hostname == inventory_hostname }}" diff --git a/zuul.d/molecule.yaml b/zuul.d/molecule.yaml index 993190334..265e691e3 100644 --- a/zuul.d/molecule.yaml +++ b/zuul.d/molecule.yaml @@ -43,6 +43,7 @@ - tripleo-ansible-centos-7-molecule-tripleo-puppet-cache - tripleo-ansible-centos-7-molecule-tripleo-systemd-wrapper - tripleo-ansible-centos-7-molecule-tripleo-keystone-resources + - tripleo-ansible-centos-7-molecule-tripleo-nova-image-cache gate: jobs: - tripleo-ansible-centos-7-molecule-aide @@ -86,6 +87,7 @@ - tripleo-ansible-centos-7-molecule-tripleo-puppet-cache - tripleo-ansible-centos-7-molecule-tripleo-systemd-wrapper - tripleo-ansible-centos-7-molecule-tripleo-keystone-resources + - tripleo-ansible-centos-7-molecule-tripleo-nova-image-cache name: tripleo-ansible-molecule-jobs - job: files: @@ -389,3 +391,12 @@ parent: tripleo-ansible-centos-7-base vars: tripleo_role_name: tripleo-keystone-resources +- job: + files: + - ^tripleo_ansible/roles/tripleo-nova-image-cache/.* + - ^tripleo_ansible/ansible_plugins/action/tripleo_nova_image_cache.py + - ^tripleo_ansible/ansible_plugins/modules/tripleo_nova_image_cache.py + name: tripleo-ansible-centos-7-molecule-tripleo-nova-image-cache + parent: tripleo-ansible-centos-7-base + vars: + tripleo_role_name: tripleo-nova-image-cache