tripleo-container-manage: introduce concurrency

Co-Authored-By: Alex Schultz <aschultz@redhat.com>
Change-Id: I1e5e941558c492b050e4db542703e322707dbbbd

tripleo_ansible/ansible_plugins/filter/helpers.py

@@ -31,19 +31,27 @@ class FilterModule(object):
     def subsort(self, dict_to_sort, attribute, null_value=None):
         """Sort a hash from a sub-element.
 
-        This filter will return a sorted list of tuples from a dictionary
+        This filter will return an dictionary ordered by the attribute
-        using an attribute from within the hash. If the sort attribute is
+        part of each item.
-        undefined it will be set in the returned item as the defined
-        `null_value`. This makes it possible to sort all items equally.
         """
         for k, v in dict_to_sort.items():
             if attribute not in v:
                 dict_to_sort[k][attribute] = null_value
 
-        return sorted(
+        data = {}
-            dict_to_sort.items(),
+        for d in dict_to_sort.items():
-            key=lambda x: x[1][attribute]
+            if d[1][attribute] not in data:
+                data[d[1][attribute]] = []
+            data[d[1][attribute]].append({d[0]: d[1]})
+
+        sorted_list = sorted(
+            data.items(),
+            key=lambda x: x[0]
         )
+        ordered_dict = {}
+        for o, v in sorted_list:
+            ordered_dict[o] = v
+        return ordered_dict
 
     def singledict(self, list_to_convert):
         """Generate a single dictionary from a list of dictionaries.

tripleo_ansible/roles/tripleo-container-manage/defaults/main.yml

@@ -19,6 +19,7 @@
 
 # All variables within this role should have a prefix of "tripleo_container_manage"
 tripleo_container_manage_cli: podman
+tripleo_container_manage_concurrency: 1
 tripleo_container_manage_config: "/var/lib/tripleo-config/"
 tripleo_container_manage_config_id: tripleo
 tripleo_container_manage_config_patterns: 'hashed-*.json'

tripleo_ansible/roles/tripleo-container-manage/tasks/create.yml

@@ -17,5 +17,5 @@
 - name: "Create containers managed by Podman for {{ tripleo_container_manage_config }}"
   when:
     - tripleo_container_manage_cli == 'podman'
-  include_tasks: podman/create.yml
+  include: podman/start_order.yml order="{{ item.key }}" data="{{ item.value }}"
-  loop: "{{ all_containers_hash | subsort(attribute='start_order', null_value=0) }}"
+  loop: "{{ all_containers_hash | subsort(attribute='start_order', null_value=0) | dict2items | list }}"

tripleo_ansible/roles/tripleo-container-manage/tasks/podman/create.yml

@@ -14,66 +14,71 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-- name: "Run execute task"
+- name: Prepare container facts
+  set_fact:
+    container_name: "{{ batched_container_data.0.keys() | first }}"
+    container_data: "{{ batched_container_data.0[batched_container_data.0.keys()|first] }}"
+
+- name: "{{ container_name }} : Run execute task"
   include_tasks: exec.yml
   when:
-    - item.1.action is defined
+    - container_data.action is defined
-    - item.1.action == 'exec'
+    - container_data.action == 'exec'
 
-- name: "Manage container for {{ item.0 }}"
+- name: "{{ container_name }} : Manage container"
-  when: item.1.action is not defined
+  when: container_data.action is not defined
   podman_container:
-    cap_add: "{{ item.1.cap_add | default(omit) }}"
+    cap_add: "{{ container_data.cap_add | default(omit) }}"
-    cap_drop: "{{ item.1.cap_drop | default(omit) }}"
+    cap_drop: "{{ container_data.cap_drop | default(omit) }}"
-    command: "{{ item.1.command | default(omit) }}"
+    command: "{{ container_data.command | default(omit) }}"
-    conmon_pidfile: "/var/run/{{ item.0 }}.pid"
+    conmon_pidfile: "/var/run/{{ container_name }}.pid"
-    cpu_shares: "{{ item.1.cpu_shares | default(omit) | int }}"
+    cpu_shares: "{{ container_data.cpu_shares | default(omit) | int }}"
-    # cpuset_cpus: "{{ item.1.cpuset_cpus | default(omit) }}"
+    # cpuset_cpus: "{{ container_data.cpuset_cpus | default(omit) }}"
-    detach: "{{ item.1.detach | default(true) }}"
+    detach: "{{ container_data.detach | default(true) }}"
-    env: "{{ item.1.environment if item.1.environment is defined and item.1.environment else {} }}"
+    env: "{{ container_data.environment if container_data.environment is defined and container_data.environment else {} }}"
-    env_file: "{{ item.1.env_file | default(omit) }}"
+    env_file: "{{ container_data.env_file | default(omit) }}"
-    etc_hosts: "{{ item.1.extra_hosts | default({}) }}"
+    etc_hosts: "{{ container_data.extra_hosts | default({}) }}"
-    group_add: "{{ item.1.group_add | default(omit) }}"
+    group_add: "{{ container_data.group_add | default(omit) }}"
-    hostname: "{{ item.1.hostname | default(omit) }}"
+    hostname: "{{ container_data.hostname | default(omit) }}"
-    image: "{{ item.1.image }}"
+    image: "{{ container_data.image }}"
-    interactive: "{{ item.1.interactive | default(false) }}"
+    interactive: "{{ container_data.interactive | default(false) }}"
-    ipc: "{{ item.1.ipc | default(omit) }}"
+    ipc: "{{ container_data.ipc | default(omit) }}"
     label:
       config_id: "{{ tripleo_container_manage_config_id }}"
-      container_name: "{{ item.0 }}"
+      container_name: "{{ container_name }}"
       managed_by: tripleo_ansible
-      config_data: "{{ item.1 | to_json }}"
+      config_data: "{{ container_data | to_json }}"
     log_driver: 'k8s-file'
     # log_opt: |
     #   "{{ 'path=' if tripleo_container_manage_log_path is defined else '' }}
     #   {{ tripleo_container_manage_log_path | default('') }}
     #   {{ '/' if tripleo_container_manage_log_path is defined else '' }}
-    #   {{ item.0 if tripleo_container_manage_log_path is defined else '' }}"
+    #   {{ container_data if tripleo_container_manage_log_path is defined else '' }}"
-    memory: "{{ item.1.mem_limit | default(omit) }}"
+    memory: "{{ container_data.mem_limit | default(omit) }}"
-    memory_swap: "{{ item.1.mem_swappiness | default(omit) }}"
+    memory_swap: "{{ container_data.mem_swappiness | default(omit) }}"
-    name: "{{ item.0 }}"
+    name: "{{ container_name }}"
-    net: "{{ item.1.net | default('none') }}"
+    net: "{{ container_data.net | default('none') }}"
-    pid: "{{ item.1.pid | default(omit) }}"
+    pid: "{{ container_data.pid | default(omit) }}"
-    privileged: "{{ item.1.privileged | default(false) }}"
+    privileged: "{{ container_data.privileged | default(false) }}"
-    rm: "{{ item.1.remove | default(false) }}"
+    rm: "{{ container_data.remove | default(false) }}"
-    security_opt: "{{ item.1.security_opt | default(omit) }}"
+    security_opt: "{{ container_data.security_opt | default(omit) }}"
     state: present
-    stop_signal: "{{ item.1.stop_signal | default(omit) }}"
+    stop_signal: "{{ container_data.stop_signal | default(omit) }}"
-    stop_timeout: "{{ item.1.stop_grace_period | default(omit) | int }}"
+    stop_timeout: "{{ container_data.stop_grace_period | default(omit) | int }}"
-    tty: "{{ item.1.tty | default(false) }}"
+    tty: "{{ container_data.tty | default(false) }}"
-    ulimit: "{{ item.1.ulimit | default(omit) }}"
+    ulimit: "{{ container_data.ulimit | default(omit) }}"
-    user: "{{ item.1.user | default(omit) }}"
+    user: "{{ container_data.user | default(omit) }}"
-    uts: "{{ item.1.uts | default(omit) }}"
+    uts: "{{ container_data.uts | default(omit) }}"
-    volume: "{{ item.1.volumes | default(omit) }}"
+    volume: "{{ container_data.volumes | default(omit) }}"
-    volumes_from: "{{ item.1.volumes_from | default([]) }}"
+    volumes_from: "{{ container_data.volumes_from | default([]) }}"
   register: podman_container
 
-- name: Manage systemd service for {{ item.0 }}
+- name: "{{ container_name }} : Manage systemd service }}"
   include_tasks: systemd.yml
   when:
-    - item.1.action is not defined
+    - container_data.action is not defined
-    - item.1.restart is defined
+    - container_data.restart is defined
     # systemd doesn't have the equivalent of docker unless-stopped.
     # Let's force 'always' so containers aren't restarted when stopped by
     # systemd, but restarted when in failure.
-    - item.1.restart == 'always' or item.1.restart == 'unless-stopped'
+    - container_data.restart == 'always' or container_data.restart == 'unless-stopped'

tripleo_ansible/roles/tripleo-container-manage/tasks/podman/exec.yml

@@ -14,23 +14,23 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-- name: "Execute a command within a running container for {{ item.0 }}"
+- name: "Execute a command within a running container for {{ container_name }}"
   check_mode: false
   block:
-    - name: "Check if {{ item.1.command.0 }} container is running"
+    - name: "Check if {{ container_data.command.0 }} container is running"
       block:
-        - name: "Fail if {{ item.1.command.0 }} is not running"
+        - name: "Fail if {{ container_data.command.0 }} is not running"
           fail:
-            msg: "Can't run container exec for {{ item.0 }}, {{ item.1.command.0 }} is not running"
+            msg: "Can't run container exec for {{ container_name }}, {{ container_data.command.0 }} is not running"
           when:
             - not podman_containers.containers.0.State.Running
-    - name: "Prepare the exec command for {{ item.0 }}"
+    - name: "Prepare the exec command for {{ container_name }}"
       set_fact:
         cmd_template:
           - "{{ tripleo_container_manage_cli }}"
           - "exec"
           - "-u"
-          - "{{ item.1.user if item.1.user is defined else 'root' }}"
+          - "{{ container_data.user if container_data.user is defined else 'root' }}"
-    - name: "Run the container exec for {{ item.0 }}"
+    - name: "Run the container exec for {{ container_name }}"
       command:
-        argv: "{{ cmd_template + item.1.command }}"
+        argv: "{{ cmd_template + container_data.command }}"

tripleo_ansible/roles/tripleo-container-manage/tasks/podman/start_order.yml

@@ -0,0 +1,21 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: "Batching items for start_order {{ order }}"
+  include_tasks: podman/create.yml
+  loop: "{{ data | batch(tripleo_container_manage_concurrency) | list }}"
+  loop_control:
+    loop_var: batched_container_data

tripleo_ansible/roles/tripleo-container-manage/tasks/podman/systemd.yml

@@ -22,55 +22,55 @@
     podman_drop_in: true
   when:
     - podman_drop_in.stat.exists
-- name: "Start systemd service for {{ item.0 }}"
+- name: "Start systemd service for {{ container_name }}"
   block:
-    - name: "Remove trailing .requires for {{ item.0 }}"
+    - name: "Remove trailing .requires for {{ container_name }}"
       file:
-        path: "/etc/systemd/system/tripleo_{{ item.0 }}.requires"
+        path: "/etc/systemd/system/tripleo_{{ container_name }}.requires"
         state: absent
-    - name: "Create systemd unit file for {{ item.0 }} service"
+    - name: "Create systemd unit file for {{ container_name }} service"
       template:
         src: systemd-service.j2
-        dest: "/etc/systemd/system/tripleo_{{ item.0 }}.service"
+        dest: "/etc/systemd/system/tripleo_{{ container_name }}.service"
         mode: '0644'
         owner: root
         group: root
       register: systemd_file
-    - name: "Enable and start systemd service for {{ item.0 }}"
+    - name: "Enable and start systemd service for {{ container_name }}"
       systemd:
         # Restart the service if it was already running
         state: restarted
-        name: "tripleo_{{ item.0 }}.service"
+        name: "tripleo_{{ container_name }}.service"
         enabled: true
         daemon_reload: true
       when:
         - systemd_file.changed or podman_container.changed
-    - name: "Manage systemd healthcheck for {{ item.0 }}"
+    - name: "Manage systemd healthcheck for {{ container_name }}"
       when:
         - not tripleo_container_manage_healthcheck_disabled
-        - item.1.healthcheck is defined
+        - container_data.healthcheck is defined
       block:
-        - name: "Create systemd unit file for {{ item.0 }} healthcheck"
+        - name: "Create systemd unit file for {{ container_name }} healthcheck"
           template:
             src: systemd-healthcheck.j2
-            dest: "/etc/systemd/system/tripleo_{{ item.0 }}_healthcheck.service"
+            dest: "/etc/systemd/system/tripleo_{{ container_name }}_healthcheck.service"
             mode: '0644'
             owner: root
             group: root
           register: systemd_healthcheck
-        - name: "Create systemd timer for {{ item.0 }} healthcheck"
+        - name: "Create systemd timer for {{ container_name }} healthcheck"
           template:
             src: systemd-timer.j2
-            dest: "/etc/systemd/system/tripleo_{{ item.0 }}_healthcheck.timer"
+            dest: "/etc/systemd/system/tripleo_{{ container_name }}_healthcheck.timer"
             mode: '0644'
             owner: root
             group: root
           register: systemd_timer
-    - name: "Enable and start systemd timer for {{ item.0 }}"
+    - name: "Enable and start systemd timer for {{ container_name }}"
       systemd:
         # Restart the timer if it was already running
         state: restarted
-        name: "tripleo_{{ item.0 }}_healthcheck.timer"
+        name: "tripleo_{{ container_name }}_healthcheck.timer"
         enabled: true
         daemon_reload: true
       when:

tripleo_ansible/roles/tripleo-container-manage/templates/systemd-healthcheck.j2

@@ -1,9 +1,9 @@
 [Unit]
-Description=tripleo_{{ item.0 }} healthcheck
+Description=tripleo_{{ container_name }} healthcheck
-After=paunch-container-shutdown.service tripleo_{{ item.0 }}.service
+After=paunch-container-shutdown.service tripleo_{{ container_name }}.service
-Requisite=tripleo_{{ item.0 }}.service
+Requisite=tripleo_{{ container_name }}.service
 [Service]
 Type=oneshot
-ExecStart=/usr/bin/podman exec {{ item.0 }} {{ item.1.healthcheck.test }}
+ExecStart=/usr/bin/podman exec {{ container_name }} {{ container_data.healthcheck.test }}
 [Install]
 WantedBy=multi-user.target

tripleo_ansible/roles/tripleo-container-manage/templates/systemd-service.j2

@@ -1,20 +1,20 @@
 [Unit]
-Description={{ item.0 }} container
+Description={{ container_name }} container
 After=paunch-container-shutdown.service
-Wants={{ item.1.depends_on | default([]) | join(',') }}
+Wants={{ container_data.depends_on | default([]) | join(',') }}
 [Service]
 Restart=always
-{% if item.1.depends_on is defined and (item.1.depends_on | length > 0) and podman_drop_in | default('false') %}
+{% if container_data.depends_on is defined and (container_data.depends_on | length > 0) and podman_drop_in | default('false') %}
-ExecStart=/usr/libexec/paunch-start-podman-container {{ item.0 }}
+ExecStart=/usr/libexec/paunch-start-podman-container {{ container_name }}
 {% else %}
-ExecStart=/usr/bin/podman start {{ item.0 }}
+ExecStart=/usr/bin/podman start {{ container_name }}
 {% endif %}
-ExecStop=/usr/bin/podman stop -t {{ item.1.stop_grace_period | default(10) | int }} {{ item.0 }}
+ExecStop=/usr/bin/podman stop -t {{ container_data.stop_grace_period | default(10) | int }} {{ container_name }}
 KillMode=none
 Type=forking
-PIDFile=/var/run/{{ item.0 }}.pid
+PIDFile=/var/run/{{ container_name }}.pid
-{% if item.1.systemd_exec_flags is defined %}
+{% if container_data.systemd_exec_flags is defined %}
-{%   for s_flag, s_value in item.1.systemd_exec_flags.items() %}
+{%   for s_flag, s_value in container_data.systemd_exec_flags.items() %}
 {{ s_flag }}={{ s_value }}
 {%   endfor %}
 {% endif %}

tripleo_ansible/roles/tripleo-container-manage/templates/systemd-timer.j2

@@ -1,9 +1,9 @@
 [Unit]
-Description=tripleo_{{ item.0 }} container healthcheck
+Description=tripleo_{{ container_name }} container healthcheck
-PartOf=tripleo_{{ item.0 }}.service
+PartOf=tripleo_{{ container_name }}.service
 [Timer]
 OnActiveSec=120
-OnUnitActiveSec={{ item.1.check_interval | default(60) }}
+OnUnitActiveSec={{ container_data.check_interval | default(60) }}
-RandomizedDelaySec={{ 45 if item.1.check_interval is not defined else (item.1.check_interval * 3 / 4) | int | abs }}
+RandomizedDelaySec={{ 45 if container_data.check_interval is not defined else (container_data.check_interval * 3 / 4) | int | abs }}
 [Install]
 WantedBy=timers.target