From 82406ad958f08da71cd76ba6cfe3a7930c744ba8 Mon Sep 17 00:00:00 2001 From: Markos Chandras Date: Fri, 16 Jun 2017 18:52:48 +0100 Subject: [PATCH] systemd: Set a higher DefaultTasksMax value systemd-2.28 introduced DefaultTasksMax which is used to control the default TasksMax= setting for services and scopes running on the system. (TasksMax= is the primary setting that exposes the "pids" cgroup controller on systemd and was introduced in the previous systemd release.) The setting now defaults to 512, which means services that are not explicitly configured otherwise will only be able to create 512 processes or threads at maximum, from this version on. However, the 512 limit seems too strict and sometimes leads to failures like the following one on busy containers ==> opensuse422: fatal: [container3]: FAILED! => {"changed": false, "cmd": "/usr/sbin/rabbitmqctl -q -n '' list_user_permissions guest", "failed": true, "msg": "/usr/sbin/rabbitmqctl: fork: retry: No child processes\n/usr/lib64/rabbitmq/lib/rabbitmq_server-3.6.6//sbin/rabbitmq-env: fork: retry: Resource temporarily unavailable\n/usr/lib64/rabbitmq/lib/rabbitmq_server-3.6.6//sbin/rabbitmq-env: fork: retry: No child processes\n/usr/lib64/rabbitmq/lib/rabbitmq_server-3.6.6//sbin/rabbitmq-env: fork: retry: No child processes\nFailed to create thread: Resource temporarily unavailable (11)\r\nAborted (core dumped)", "rc": 134, "stderr": "/usr/sbin/rabbitmqctl: fork: retry: No child processes\n/usr/lib64/rabbitmq/lib/rabbitmq_server-3.6.6//sbin/rabbitmq-env: fork: retry: Resource temporarily unavailable\n/usr/lib64/rabbitmq/lib/rabbitmq_server-3.6.6//sbin/rabbitmq-env: fork: retry: No child processes\n/usr/lib64/rabbitmq/lib/rabbitmq_server-3.6.6//sbin/rabbitmq-env: fork: retry: No child processes\nFailed to create thread: Resource temporarily unavailable (11)\r\nAborted (core dumped)\n", "stderr_lines": ["/usr/sbin/rabbitmqctl: fork: retry: No child processes", "/usr/lib64/rabbitmq/lib/rabbitmq_server-3.6.6//sbin/rabbitmq-env: fork: retry: Resource temporarily unavailable", "/usr/lib64/rabbitmq/lib/rabbitmq_server-3.6.6//sbin/rabbitmq-env: fork: retry: No child processes", "/usr/lib64/rabbitmq/lib/rabbitmq_server-3.6.6//sbin/rabbitmq-env: fork: retry: No child processes", "Failed to create thread: Resource temporarily unavailable (11)", "Aborted (core dumped)"], "stdout": "", "stdout_lines": []} and with messages in the kernel log such as [ 2925.999021] cgroup: fork rejected by pids controller in /init.scope/lxc/container1 [ 3083.704049] cgroup: fork rejected by pids controller in /init.scope/lxc/container2 As we see, even though the /init.scope/lxc/container1 as pids.max set to 'max', the /init.scope has pids.max set to 512 and in cgroups we always respect the lowest boundary ~> cat /sys/fs/cgroup/pids/init.scope/lxc/container1/pids.max max ~> cat /sys/fs/cgroup/pids/init.scope/pids.max 512 As a result of which, the 512 limit is enforced. As such, we add a new variable to make this limit configurable. The default limit has now been increased to 8192. Change-Id: I8b4143aac84d4c795cab9c0d978c9a97ebea1793 --- defaults/main.yml | 4 ++++ handlers/main.yml | 7 +++++++ tasks/lxc_post_install.yml | 13 +++++++++++++ 3 files changed, 24 insertions(+) diff --git a/defaults/main.yml b/defaults/main.yml index b7aed348..becf6b2f 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -31,6 +31,10 @@ lxc_architecture_mapping: # Set the volume size in gigabytes for the machine image caches. lxc_host_machine_volume_size: 16 +# DefaultTasksMax systemd value. It's not recommended to change this value as it +# could prevent new processes from starting on busy containers. +lxc_default_tasks_max: 8192 + # lxc container rootfs directory and cache path lxc_container_directory: "/var/lib/lxc" lxc_container_cache_path: "/var/cache/lxc/download" diff --git a/handlers/main.yml b/handlers/main.yml index e6c1878c..de51d416 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -44,3 +44,10 @@ - name: Veth check command: "/usr/local/bin/lxc-veth-check" + +- name: Reload systemd units + systemd: + # TODO(hwoarang): We only want to reload the units so the 'name' + # parameter can be removed when we move to Ansible 2.4 + name: it_does_not_matter + daemon_reload: yes diff --git a/tasks/lxc_post_install.yml b/tasks/lxc_post_install.yml index 76e6a13a..6122752b 100644 --- a/tasks/lxc_post_install.yml +++ b/tasks/lxc_post_install.yml @@ -50,6 +50,19 @@ - lxc-files - lxc-config +- name: Set systemd DefaultTasksMax value + lineinfile: + dest: /etc/systemd/system.conf + state: present + regexp: "^.*DefaultTasksMax.*$" + line: "DefaultTasksMax={{ lxc_default_tasks_max }} # Managed by Ansible" + when: + - ansible_service_mgr == 'systemd' + notify: + - Reload systemd units + tags: + - lxc-config + # Ensure apparmor reindex runs before other things that may fail - meta: flush_handlers