From e26f817597cf9f5a08d2eb6d559d1f2cc1b7d031 Mon Sep 17 00:00:00 2001 From: Emilien Macchi Date: Wed, 6 Nov 2019 17:31:28 +0100 Subject: [PATCH] tripleo-container-manage: use async - Add a new filter 'haskey' which returns container data with a specific config key. The filter takes a list of dicts and returns the dicts which have a certain key given in parameter with 'attribute'. If 'reverse' is set to True, the returned list won't contains dicts which have the 'attribute' value. If 'any' is set to True, the returned list will match any value in the list of 'value' parameter which has to be a list. - Make the exec and create playbook using ansible "async". (WIP is to re-add the podman_container.changed in systemd playbook). Note: we don't manage Systemd resources with async, since it exposes race conditions at the systemd level. Change-Id: Ice92cd5f90039e685c171e9035f929349a67ff2c --- .../ansible_plugins/filter/helpers.py | 42 ++++++- .../tasks/container_running.yml | 27 +++++ .../tasks/podman/create.yml | 107 +++++++++--------- .../tasks/podman/exec.yml | 58 ++++++---- .../tasks/podman/manage.yml | 33 ++++++ .../tasks/podman/start_order.yml | 2 +- .../tasks/podman/systemd.yml | 11 +- 7 files changed, 195 insertions(+), 85 deletions(-) create mode 100644 tripleo_ansible/roles/tripleo-container-manage/tasks/container_running.yml create mode 100644 tripleo_ansible/roles/tripleo-container-manage/tasks/podman/manage.yml diff --git a/tripleo_ansible/ansible_plugins/filter/helpers.py b/tripleo_ansible/ansible_plugins/filter/helpers.py index e1836ed3f..605360c90 100644 --- a/tripleo_ansible/ansible_plugins/filter/helpers.py +++ b/tripleo_ansible/ansible_plugins/filter/helpers.py @@ -25,7 +25,8 @@ class FilterModule(object): return { 'singledict': self.singledict, 'subsort': self.subsort, - 'needs_delete': self.needs_delete + 'needs_delete': self.needs_delete, + 'haskey': self.haskey } def subsort(self, dict_to_sort, attribute, null_value=None): @@ -115,7 +116,12 @@ class FilterModule(object): for c in container_infos if c_name == c['Name']] except KeyError: continue - c_facts = c_facts[0] if len(c_facts) == 1 else {} + + # Build c_facts so it can be compared later with config_data; + # both will be json.dumps objects. + c_facts = json.dumps( + json.loads(c_facts[0]).get(c_name) + ) if len(c_facts) == 1 else {} # 0 was picked since it's the null_value for the subsort filter. # When a container config doesn't provide the start_order, it'll be @@ -131,3 +137,35 @@ class FilterModule(object): continue return to_delete + + def haskey(self, batched_container_data, attribute, value=None, + reverse=False, any=False): + """Return container data with a specific config key. + + This filter will take a list of dictionaries (batched_container_data) + and will return the dictionnaries which have a certain key given + in parameter with 'attribute'. + If reverse is set to True, the returned list won't contain dictionaries + which have the attribute. + If any is set to True, the returned list will match any value in + the list of values for "value" parameter which has to be a list. + """ + return_list = [] + for container in batched_container_data: + for k, v in json.loads(json.dumps(container)).items(): + if attribute in v and not reverse: + if value is None: + return_list.append({k: v}) + else: + if isinstance(value, list) and any: + if v[attribute] in value: + return_list.append({k: v}) + elif any: + raise TypeError("value has to be a list if any is " + "set to True.") + else: + if v[attribute] == value: + return_list.append({k: v}) + if attribute not in v and reverse: + return_list.append({k: v}) + return return_list diff --git a/tripleo_ansible/roles/tripleo-container-manage/tasks/container_running.yml b/tripleo_ansible/roles/tripleo-container-manage/tasks/container_running.yml new file mode 100644 index 000000000..e523b9b68 --- /dev/null +++ b/tripleo_ansible/roles/tripleo-container-manage/tasks/container_running.yml @@ -0,0 +1,27 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +- name: "Get {{ lookup('dict', container_data).value.command.0 }} container status" + set_fact: + container_running: >- + {{ podman_containers.containers | selectattr('Name', 'equalto', lookup('dict', container_data).value.command.0) | + map(attribute='State.Running') | first | default(false) }} + +- name: "Fail if {{ lookup('dict', container_data).key }} is not running" + fail: + msg: "Can't run container exec for {{ lookup('dict', container_data).key }}, {{ lookup('dict', container_data).value.command.0 }} is not running" + when: + - not container_running|default(false)|bool diff --git a/tripleo_ansible/roles/tripleo-container-manage/tasks/podman/create.yml b/tripleo_ansible/roles/tripleo-container-manage/tasks/podman/create.yml index fc6b6ac75..284d09ebd 100644 --- a/tripleo_ansible/roles/tripleo-container-manage/tasks/podman/create.yml +++ b/tripleo_ansible/roles/tripleo-container-manage/tasks/podman/create.yml @@ -14,67 +14,62 @@ # License for the specific language governing permissions and limitations # under the License. -- name: Prepare container facts - set_fact: - container_name: "{{ batched_container_data.0.keys() | first }}" - container_data: "{{ batched_container_data.0[batched_container_data.0.keys()|first] }}" - -- name: "{{ container_name }} : Run execute task" - include_tasks: exec.yml - when: - - container_data.action is defined - - container_data.action == 'exec' - -- name: "{{ container_name }} : Manage container" - when: container_data.action is not defined +# TODO(emilien) Re-add podman_container.changed from async +- name: "Async container create/run" + async: 300 + poll: 0 + register: create_async_results + loop: "{{ batched_container_data | haskey(attribute='action', reverse=True) }}" + loop_control: + loop_var: container_data podman_container: - cap_add: "{{ container_data.cap_add | default(omit) }}" - cap_drop: "{{ container_data.cap_drop | default(omit) }}" - command: "{{ container_data.command | default(omit) }}" - conmon_pidfile: "/var/run/{{ container_name }}.pid" - cpu_shares: "{{ container_data.cpu_shares | default(omit) | int }}" - # cpuset_cpus: "{{ container_data.cpuset_cpus | default(omit) }}" - detach: "{{ container_data.detach | default(true) }}" - env: "{{ container_data.environment | default({}, true) }}" - env_file: "{{ container_data.env_file | default(omit) }}" - etc_hosts: "{{ container_data.extra_hosts | default(omit) }}" - group_add: "{{ container_data.group_add | default(omit) }}" - hostname: "{{ container_data.hostname | default(omit) }}" - image: "{{ container_data.image }}" - interactive: "{{ container_data.interactive | default(false) }}" - ipc: "{{ container_data.ipc | default(omit) }}" + cap_add: "{{ lookup('dict', container_data).value.cap_add | default(omit) }}" + cap_drop: "{{ lookup('dict', container_data).value.cap_drop | default(omit) }}" + command: "{{ lookup('dict', container_data).value.command | default(omit) }}" + conmon_pidfile: "/var/run/{{ lookup('dict', container_data).key }}.pid" + cpu_shares: "{{ lookup('dict', container_data).value.cpu_shares | default(omit) | int }}" + # cpuset_cpus: "{{ lookup('dict', container_data).value.cpuset_cpus | default(omit) }}" + detach: "{{ lookup('dict', container_data).value.detach | default(true) }}" + env: "{{ lookup('dict', container_data).value.environment | default(omit) }}" + env_file: "{{ lookup('dict', container_data).value.env_file | default(omit) }}" + etc_hosts: "{{ lookup('dict', container_data).value.extra_hosts | default({}) }}" + group_add: "{{ lookup('dict', container_data).value.group_add | default(omit) }}" + hostname: "{{ lookup('dict', container_data).value.hostname | default(omit) }}" + image: "{{ lookup('dict', container_data).value.image }}" + interactive: "{{ lookup('dict', container_data).value.interactive | default(false) }}" + ipc: "{{ lookup('dict', container_data).value.ipc | default(omit) }}" label: config_id: "{{ tripleo_container_manage_config_id }}" - container_name: "{{ container_name }}" + container_name: "{{ lookup('dict', container_data).key }}" managed_by: tripleo_ansible config_data: "{{ container_data | to_json }}" log_driver: 'k8s-file' - log_opt: "path={{ tripleo_container_manage_log_path }}/{{ container_name }}.log" - memory: "{{ container_data.mem_limit | default(omit) }}" - memory_swap: "{{ container_data.mem_swappiness | default(omit) }}" - name: "{{ container_name }}" - net: "{{ container_data.net | default('none') }}" - pid: "{{ container_data.pid | default(omit) }}" - privileged: "{{ container_data.privileged | default(false) }}" - rm: "{{ container_data.remove | default(false) }}" - security_opt: "{{ container_data.security_opt | default(omit) }}" + log_opt: "path={{ tripleo_container_manage_log_path }}/{{ lookup('dict', container_data).key }}.log" + memory: "{{ lookup('dict', container_data).value.mem_limit | default(omit) }}" + memory_swap: "{{ lookup('dict', container_data).value.mem_swappiness | default(omit) }}" + name: "{{ lookup('dict', container_data).key }}" + net: "{{ lookup('dict', container_data).value.net | default('none') }}" + pid: "{{ lookup('dict', container_data).value.pid | default(omit) }}" + privileged: "{{ lookup('dict', container_data).value.privileged | default(false) }}" + rm: "{{ lookup('dict', container_data).value.remove | default(false) }}" + security_opt: "{{ lookup('dict', container_data).value.security_opt | default(omit) }}" state: present - stop_signal: "{{ container_data.stop_signal | default(omit) }}" - stop_timeout: "{{ container_data.stop_grace_period | default(omit) | int }}" - tty: "{{ container_data.tty | default(false) }}" - ulimit: "{{ container_data.ulimit | default(omit) }}" - user: "{{ container_data.user | default(omit) }}" - uts: "{{ container_data.uts | default(omit) }}" - volume: "{{ container_data.volumes | default(omit) }}" - volumes_from: "{{ container_data.volumes_from | default(omit) }}" - register: podman_container + stop_signal: "{{ lookup('dict', container_data).value.stop_signal | default(omit) }}" + stop_timeout: "{{ lookup('dict', container_data).value.stop_grace_period | default(omit) | int }}" + tty: "{{ lookup('dict', container_data).value.tty | default(false) }}" + ulimit: "{{ lookup('dict', container_data).value.ulimit | default(omit) }}" + user: "{{ lookup('dict', container_data).value.user | default(omit) }}" + uts: "{{ lookup('dict', container_data).value.uts | default(omit) }}" + volume: "{{ lookup('dict', container_data).value.volumes | default(omit) }}" + volumes_from: "{{ lookup('dict', container_data).value.volumes_from | default([]) }}" -- name: "{{ container_name }} : Manage systemd service }}" - include_tasks: systemd.yml - when: - - container_data.action is not defined - - container_data.restart is defined - # systemd doesn't have the equivalent of docker unless-stopped. - # Let's force 'always' so containers aren't restarted when stopped by - # systemd, but restarted when in failure. - - container_data.restart == 'always' or container_data.restart == 'unless-stopped' +- name: "Check podman create status" + async_status: + jid: "{{ create_async_result_item.ansible_job_id }}" + loop: "{{ create_async_results.results }}" + loop_control: + loop_var: "create_async_result_item" + register: create_async_poll_results + until: create_async_poll_results.finished + retries: 30 + when: not ansible_check_mode|bool diff --git a/tripleo_ansible/roles/tripleo-container-manage/tasks/podman/exec.yml b/tripleo_ansible/roles/tripleo-container-manage/tasks/podman/exec.yml index 1220253a4..ad8d96494 100644 --- a/tripleo_ansible/roles/tripleo-container-manage/tasks/podman/exec.yml +++ b/tripleo_ansible/roles/tripleo-container-manage/tasks/podman/exec.yml @@ -14,28 +14,36 @@ # License for the specific language governing permissions and limitations # under the License. -- name: "Execute a command within a running container for {{ container_name }}" - check_mode: false - block: - - name: "Check if {{ container_data.command.0 }} container is running" - block: - - name: Get container status - set_fact: - container_running: >- - {{ podman_containers.containers | selectattr('Name', 'equalto', container_data.command.0 ) | - map(attribute='State.Running') | first | default(false) }} - - name: "Fail if {{ container_data.command.0 }} is not running" - fail: - msg: "Can't run container exec for {{ container_name }}, {{ container_data.command.0 }} is not running" - when: - - not container_running|bool - - name: "Prepare the exec command for {{ container_name }}" - set_fact: - cmd_template: - - "{{ tripleo_container_manage_cli }}" - - "exec" - - "-u" - - "{{ container_data.user if container_data.user is defined else 'root' }}" - - name: "Run the container exec for {{ container_name }}" - command: - argv: "{{ cmd_template + container_data.command }}" +- name: "Check if containers are running before doing exec" + include_tasks: container_running.yml + loop: "{{ batched_container_data | haskey(attribute='action', value='exec') }}" + loop_control: + loop_var: container_data + +- name: "Run actions async" + command: + argv: "{{ cmd_template + lookup('dict', container_data).value.command }}" + vars: + cmd_template: + - "{{ tripleo_container_manage_cli }}" + - "exec" + - "-u" + - "{{ lookup('dict', container_data).value.user if lookup('dict', container_data).value.user is defined else 'root' }}" + async: 60 + poll: 0 + register: exec_async_results + loop: "{{ batched_container_data | haskey(attribute='action', value='exec') }}" + loop_control: + loop_var: container_data + when: not ansible_check_mode|bool + +- name: "Check podman exec status" + async_status: + jid: "{{ exec_async_result_item.ansible_job_id }}" + loop: "{{ exec_async_results.results }}" + loop_control: + loop_var: "exec_async_result_item" + register: exec_async_poll_results + until: exec_async_poll_results.finished + retries: 30 + when: not ansible_check_mode|bool diff --git a/tripleo_ansible/roles/tripleo-container-manage/tasks/podman/manage.yml b/tripleo_ansible/roles/tripleo-container-manage/tasks/podman/manage.yml new file mode 100644 index 000000000..8427f8043 --- /dev/null +++ b/tripleo_ansible/roles/tripleo-container-manage/tasks/podman/manage.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Run containers execs asynchronously + include_tasks: podman/exec.yml + +- name: Manage containers asynchronously + include_tasks: podman/create.yml + +# We don't want to use async for the systemd tasks or we can have startup +# errors when systemd has to deal with multiple services trying to start +# at the same time. It is more reliable to start them in serial. +- name: Manage container systemd services and healthchecks in serial + include_tasks: podman/systemd.yml + # systemd doesn't have the equivalent of docker unless-stopped. + # Let's force 'always' so containers aren't restarted when stopped by + # systemd, but restarted when in failure. + loop: "{{ batched_container_data | haskey(attribute='restart', value=['always','unless-stopped'], any=True) }}" + loop_control: + loop_var: container_config diff --git a/tripleo_ansible/roles/tripleo-container-manage/tasks/podman/start_order.yml b/tripleo_ansible/roles/tripleo-container-manage/tasks/podman/start_order.yml index 69826f11e..7bf328f0e 100644 --- a/tripleo_ansible/roles/tripleo-container-manage/tasks/podman/start_order.yml +++ b/tripleo_ansible/roles/tripleo-container-manage/tasks/podman/start_order.yml @@ -21,7 +21,7 @@ - tripleo_container_manage_cli == 'podman' - name: "Batching items for start_order {{ order }}" - include_tasks: podman/create.yml + include_tasks: podman/manage.yml loop: "{{ data | batch(tripleo_container_manage_concurrency) | list }}" loop_control: loop_var: batched_container_data diff --git a/tripleo_ansible/roles/tripleo-container-manage/tasks/podman/systemd.yml b/tripleo_ansible/roles/tripleo-container-manage/tasks/podman/systemd.yml index 31c0dee86..dcf8d1a79 100644 --- a/tripleo_ansible/roles/tripleo-container-manage/tasks/podman/systemd.yml +++ b/tripleo_ansible/roles/tripleo-container-manage/tasks/podman/systemd.yml @@ -13,15 +13,23 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + - name: Check if /etc/sysconfig/podman_drop_in exists stat: path: /etc/sysconfig/podman_drop_in register: podman_drop_in + - name: Set podman_drop_in fact set_fact: podman_drop_in: true when: - podman_drop_in.stat.exists + +- name: Set container_name and container_data facts + set_fact: + container_name: "{{ lookup('dict', container_config).key }}" + container_data: "{{ lookup('dict', container_config).value }}" + - name: "Start systemd service for {{ container_name }}" block: - name: "Remove trailing .requires for {{ container_name }}" @@ -44,7 +52,8 @@ enabled: true daemon_reload: true when: - - systemd_file.changed or podman_container.changed + # Re-add podman_container.changed from async + - systemd_file.changed - name: "Manage systemd healthcheck for {{ container_name }}" when: - not tripleo_container_manage_healthcheck_disabled