tripleo-container-manage: use async

- Add a new filter 'haskey' which returns container data with a specific
  config key. The filter takes a list of dicts and returns the dicts
  which have a certain key given in parameter with 'attribute'.
  If 'reverse' is set to True, the returned list won't contains dicts
  which have the 'attribute' value.
  If 'any' is set to True, the returned list will match any value in the
  list of 'value' parameter which has to be a list.

- Make the exec and create playbook using ansible "async".
  (WIP is to re-add the podman_container.changed in systemd playbook).

Note: we don't manage Systemd resources with async, since it exposes
race conditions at the systemd level.

Change-Id: Ice92cd5f90039e685c171e9035f929349a67ff2c

tripleo_ansible/ansible_plugins/filter/helpers.py

@@ -25,7 +25,8 @@ class FilterModule(object):
         return {
             'singledict': self.singledict,
             'subsort': self.subsort,
-            'needs_delete': self.needs_delete
+            'needs_delete': self.needs_delete,
+            'haskey': self.haskey
         }
 
     def subsort(self, dict_to_sort, attribute, null_value=None):
@@ -115,7 +116,12 @@ class FilterModule(object):
                            for c in container_infos if c_name == c['Name']]
             except KeyError:
                 continue
-            c_facts = c_facts[0] if len(c_facts) == 1 else {}
+
+            # Build c_facts so it can be compared later with config_data;
+            # both will be json.dumps objects.
+            c_facts = json.dumps(
+                json.loads(c_facts[0]).get(c_name)
+            ) if len(c_facts) == 1 else {}
 
             # 0 was picked since it's the null_value for the subsort filter.
             # When a container config doesn't provide the start_order, it'll be
@@ -131,3 +137,35 @@ class FilterModule(object):
                 continue
 
         return to_delete
+
+    def haskey(self, batched_container_data, attribute, value=None,
+               reverse=False, any=False):
+        """Return container data with a specific config key.
+
+        This filter will take a list of dictionaries (batched_container_data)
+        and will return the dictionnaries which have a certain key given
+        in parameter with 'attribute'.
+        If reverse is set to True, the returned list won't contain dictionaries
+        which have the attribute.
+        If any is set to True, the returned list will match any value in
+        the list of values for "value" parameter which has to be a list.
+        """
+        return_list = []
+        for container in batched_container_data:
+            for k, v in json.loads(json.dumps(container)).items():
+                if attribute in v and not reverse:
+                    if value is None:
+                        return_list.append({k: v})
+                    else:
+                        if isinstance(value, list) and any:
+                            if v[attribute] in value:
+                                return_list.append({k: v})
+                        elif any:
+                            raise TypeError("value has to be a list if any is "
+                                            "set to True.")
+                        else:
+                            if v[attribute] == value:
+                                return_list.append({k: v})
+                if attribute not in v and reverse:
+                    return_list.append({k: v})
+        return return_list

tripleo_ansible/roles/tripleo-container-manage/tasks/container_running.yml

@@ -0,0 +1,27 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+- name: "Get {{ lookup('dict', container_data).value.command.0 }} container status"
+  set_fact:
+    container_running: >-
+      {{ podman_containers.containers | selectattr('Name', 'equalto', lookup('dict', container_data).value.command.0) |
+      map(attribute='State.Running') | first | default(false) }}
+
+- name: "Fail if {{ lookup('dict', container_data).key }} is not running"
+  fail:
+    msg: "Can't run container exec for {{ lookup('dict', container_data).key }}, {{ lookup('dict', container_data).value.command.0 }} is not running"
+  when:
+    - not container_running|default(false)|bool

tripleo_ansible/roles/tripleo-container-manage/tasks/podman/create.yml

@@ -14,67 +14,62 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-- name: Prepare container facts
+# TODO(emilien) Re-add podman_container.changed from async
-  set_fact:
+- name: "Async container create/run"
-    container_name: "{{ batched_container_data.0.keys() | first }}"
+  async: 300
-    container_data: "{{ batched_container_data.0[batched_container_data.0.keys()|first] }}"
+  poll: 0
-
+  register: create_async_results
-- name: "{{ container_name }} : Run execute task"
+  loop: "{{ batched_container_data | haskey(attribute='action', reverse=True) }}"
-  include_tasks: exec.yml
+  loop_control:
-  when:
+    loop_var: container_data
-    - container_data.action is defined
-    - container_data.action == 'exec'
-
-- name: "{{ container_name }} : Manage container"
-  when: container_data.action is not defined
   podman_container:
-    cap_add: "{{ container_data.cap_add | default(omit) }}"
+    cap_add: "{{ lookup('dict', container_data).value.cap_add | default(omit) }}"
-    cap_drop: "{{ container_data.cap_drop | default(omit) }}"
+    cap_drop: "{{ lookup('dict', container_data).value.cap_drop | default(omit) }}"
-    command: "{{ container_data.command | default(omit) }}"
+    command: "{{ lookup('dict', container_data).value.command | default(omit) }}"
-    conmon_pidfile: "/var/run/{{ container_name }}.pid"
+    conmon_pidfile: "/var/run/{{ lookup('dict', container_data).key }}.pid"
-    cpu_shares: "{{ container_data.cpu_shares | default(omit) | int }}"
+    cpu_shares: "{{ lookup('dict', container_data).value.cpu_shares | default(omit) | int }}"
-    # cpuset_cpus: "{{ container_data.cpuset_cpus | default(omit) }}"
+    # cpuset_cpus: "{{ lookup('dict', container_data).value.cpuset_cpus | default(omit) }}"
-    detach: "{{ container_data.detach | default(true) }}"
+    detach: "{{ lookup('dict', container_data).value.detach | default(true) }}"
-    env: "{{ container_data.environment | default({}, true) }}"
+    env: "{{ lookup('dict', container_data).value.environment | default(omit) }}"
-    env_file: "{{ container_data.env_file | default(omit) }}"
+    env_file: "{{ lookup('dict', container_data).value.env_file | default(omit) }}"
-    etc_hosts: "{{ container_data.extra_hosts | default(omit) }}"
+    etc_hosts: "{{ lookup('dict', container_data).value.extra_hosts | default({}) }}"
-    group_add: "{{ container_data.group_add | default(omit) }}"
+    group_add: "{{ lookup('dict', container_data).value.group_add | default(omit) }}"
-    hostname: "{{ container_data.hostname | default(omit) }}"
+    hostname: "{{ lookup('dict', container_data).value.hostname | default(omit) }}"
-    image: "{{ container_data.image }}"
+    image: "{{ lookup('dict', container_data).value.image }}"
-    interactive: "{{ container_data.interactive | default(false) }}"
+    interactive: "{{ lookup('dict', container_data).value.interactive | default(false) }}"
-    ipc: "{{ container_data.ipc | default(omit) }}"
+    ipc: "{{ lookup('dict', container_data).value.ipc | default(omit) }}"
     label:
       config_id: "{{ tripleo_container_manage_config_id }}"
-      container_name: "{{ container_name }}"
+      container_name: "{{ lookup('dict', container_data).key }}"
       managed_by: tripleo_ansible
       config_data: "{{ container_data | to_json }}"
     log_driver: 'k8s-file'
-    log_opt: "path={{ tripleo_container_manage_log_path }}/{{ container_name }}.log"
+    log_opt: "path={{ tripleo_container_manage_log_path }}/{{ lookup('dict', container_data).key }}.log"
-    memory: "{{ container_data.mem_limit | default(omit) }}"
+    memory: "{{ lookup('dict', container_data).value.mem_limit | default(omit) }}"
-    memory_swap: "{{ container_data.mem_swappiness | default(omit) }}"
+    memory_swap: "{{ lookup('dict', container_data).value.mem_swappiness | default(omit) }}"
-    name: "{{ container_name }}"
+    name: "{{ lookup('dict', container_data).key }}"
-    net: "{{ container_data.net | default('none') }}"
+    net: "{{ lookup('dict', container_data).value.net | default('none') }}"
-    pid: "{{ container_data.pid | default(omit) }}"
+    pid: "{{ lookup('dict', container_data).value.pid | default(omit) }}"
-    privileged: "{{ container_data.privileged | default(false) }}"
+    privileged: "{{ lookup('dict', container_data).value.privileged | default(false) }}"
-    rm: "{{ container_data.remove | default(false) }}"
+    rm: "{{ lookup('dict', container_data).value.remove | default(false) }}"
-    security_opt: "{{ container_data.security_opt | default(omit) }}"
+    security_opt: "{{ lookup('dict', container_data).value.security_opt | default(omit) }}"
     state: present
-    stop_signal: "{{ container_data.stop_signal | default(omit) }}"
+    stop_signal: "{{ lookup('dict', container_data).value.stop_signal | default(omit) }}"
-    stop_timeout: "{{ container_data.stop_grace_period | default(omit) | int }}"
+    stop_timeout: "{{ lookup('dict', container_data).value.stop_grace_period | default(omit) | int }}"
-    tty: "{{ container_data.tty | default(false) }}"
+    tty: "{{ lookup('dict', container_data).value.tty | default(false) }}"
-    ulimit: "{{ container_data.ulimit | default(omit) }}"
+    ulimit: "{{ lookup('dict', container_data).value.ulimit | default(omit) }}"
-    user: "{{ container_data.user | default(omit) }}"
+    user: "{{ lookup('dict', container_data).value.user | default(omit) }}"
-    uts: "{{ container_data.uts | default(omit) }}"
+    uts: "{{ lookup('dict', container_data).value.uts | default(omit) }}"
-    volume: "{{ container_data.volumes | default(omit) }}"
+    volume: "{{ lookup('dict', container_data).value.volumes | default(omit) }}"
-    volumes_from: "{{ container_data.volumes_from | default(omit) }}"
+    volumes_from: "{{ lookup('dict', container_data).value.volumes_from | default([]) }}"
-  register: podman_container
 
-- name: "{{ container_name }} : Manage systemd service }}"
+- name: "Check podman create status"
-  include_tasks: systemd.yml
+  async_status:
-  when:
+    jid: "{{ create_async_result_item.ansible_job_id }}"
-    - container_data.action is not defined
+  loop: "{{ create_async_results.results }}"
-    - container_data.restart is defined
+  loop_control:
-    # systemd doesn't have the equivalent of docker unless-stopped.
+    loop_var: "create_async_result_item"
-    # Let's force 'always' so containers aren't restarted when stopped by
+  register: create_async_poll_results
-    # systemd, but restarted when in failure.
+  until: create_async_poll_results.finished
-    - container_data.restart == 'always' or container_data.restart == 'unless-stopped'
+  retries: 30
+  when: not ansible_check_mode|bool

tripleo_ansible/roles/tripleo-container-manage/tasks/podman/exec.yml

@@ -14,28 +14,36 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-- name: "Execute a command within a running container for {{ container_name }}"
+- name: "Check if containers are running before doing exec"
-  check_mode: false
+  include_tasks: container_running.yml
-  block:
+  loop: "{{ batched_container_data | haskey(attribute='action', value='exec') }}"
-    - name: "Check if {{ container_data.command.0 }} container is running"
+  loop_control:
-      block:
+    loop_var: container_data
-        - name: Get container status
+
-          set_fact:
+- name: "Run actions async"
-            container_running: >-
+  command:
-              {{ podman_containers.containers | selectattr('Name', 'equalto', container_data.command.0 ) |
+    argv: "{{ cmd_template + lookup('dict', container_data).value.command }}"
-              map(attribute='State.Running') | first | default(false) }}
+  vars:
-        - name: "Fail if {{ container_data.command.0 }} is not running"
+    cmd_template:
-          fail:
+      - "{{ tripleo_container_manage_cli }}"
-            msg: "Can't run container exec for {{ container_name }}, {{ container_data.command.0 }} is not running"
+      - "exec"
-          when:
+      - "-u"
-            - not container_running|bool
+      - "{{ lookup('dict', container_data).value.user if lookup('dict', container_data).value.user is defined else 'root' }}"
-    - name: "Prepare the exec command for {{ container_name }}"
+  async: 60
-      set_fact:
+  poll: 0
-        cmd_template:
+  register: exec_async_results
-          - "{{ tripleo_container_manage_cli }}"
+  loop: "{{ batched_container_data | haskey(attribute='action', value='exec') }}"
-          - "exec"
+  loop_control:
-          - "-u"
+    loop_var: container_data
-          - "{{ container_data.user if container_data.user is defined else 'root' }}"
+  when: not ansible_check_mode|bool
-    - name: "Run the container exec for {{ container_name }}"
+
-      command:
+- name: "Check podman exec status"
-        argv: "{{ cmd_template + container_data.command }}"
+  async_status:
+    jid: "{{ exec_async_result_item.ansible_job_id }}"
+  loop: "{{ exec_async_results.results }}"
+  loop_control:
+    loop_var: "exec_async_result_item"
+  register: exec_async_poll_results
+  until: exec_async_poll_results.finished
+  retries: 30
+  when: not ansible_check_mode|bool

tripleo_ansible/roles/tripleo-container-manage/tasks/podman/manage.yml

@@ -0,0 +1,33 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Run containers execs asynchronously
+  include_tasks: podman/exec.yml
+
+- name: Manage containers asynchronously
+  include_tasks: podman/create.yml
+
+# We don't want to use async for the systemd tasks or we can have startup
+# errors when systemd has to deal with multiple services trying to start
+# at the same time. It is more reliable to start them in serial.
+- name: Manage container systemd services and healthchecks in serial
+  include_tasks: podman/systemd.yml
+  # systemd doesn't have the equivalent of docker unless-stopped.
+  # Let's force 'always' so containers aren't restarted when stopped by
+  # systemd, but restarted when in failure.
+  loop: "{{ batched_container_data | haskey(attribute='restart', value=['always','unless-stopped'], any=True) }}"
+  loop_control:
+    loop_var: container_config

tripleo_ansible/roles/tripleo-container-manage/tasks/podman/start_order.yml

@@ -21,7 +21,7 @@
     - tripleo_container_manage_cli == 'podman'
 
 - name: "Batching items for start_order {{ order }}"
-  include_tasks: podman/create.yml
+  include_tasks: podman/manage.yml
   loop: "{{ data | batch(tripleo_container_manage_concurrency) | list }}"
   loop_control:
     loop_var: batched_container_data

tripleo_ansible/roles/tripleo-container-manage/tasks/podman/systemd.yml

@@ -13,15 +13,23 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
+
 - name: Check if /etc/sysconfig/podman_drop_in exists
   stat:
     path: /etc/sysconfig/podman_drop_in
   register: podman_drop_in
+
 - name: Set podman_drop_in fact
   set_fact:
     podman_drop_in: true
   when:
     - podman_drop_in.stat.exists
+
+- name: Set container_name and container_data facts
+  set_fact:
+    container_name: "{{ lookup('dict', container_config).key }}"
+    container_data: "{{ lookup('dict', container_config).value }}"
+
 - name: "Start systemd service for {{ container_name }}"
   block:
     - name: "Remove trailing .requires for {{ container_name }}"
@@ -44,7 +52,8 @@
         enabled: true
         daemon_reload: true
       when:
-        - systemd_file.changed or podman_container.changed
+        # Re-add podman_container.changed from async
+        - systemd_file.changed
     - name: "Manage systemd healthcheck for {{ container_name }}"
       when:
         - not tripleo_container_manage_healthcheck_disabled