From 9c9efd9eb50be1ef35d7532df1f1c640d7e34a2d Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Thu, 14 Feb 2019 08:50:29 -0600 Subject: [PATCH] Change the q_mem and h_mem to lower and upper limits This change removes the {h,q}_mem options in favor of a new variable which clearly states the upper and lower limits for a given deployment. This change also makes these options a lot more conservative by default which will allow the deployment to better run on shared infra. Change-Id: I169f457198c11edc4881a04df65312f6c4f67feb Signed-off-by: Kevin Carter --- .../roles/elastic_dependencies/defaults/main.yml | 6 ------ .../roles/elastic_dependencies/vars/vars_elasticsearch.yml | 2 +- .../roles/elastic_dependencies/vars/vars_logstash.yml | 2 +- elk_metrics_6x/roles/elastic_logstash/tasks/main.yml | 4 ++-- .../roles/elastic_logstash/templates/logstash.yml.j2 | 2 +- elk_metrics_6x/tests/test-vars.yml | 4 ++-- elk_metrics_6x/vars/variables.yml | 7 ++----- 7 files changed, 9 insertions(+), 18 deletions(-) diff --git a/elk_metrics_6x/roles/elastic_dependencies/defaults/main.yml b/elk_metrics_6x/roles/elastic_dependencies/defaults/main.yml index 8554a42f..b1d47b71 100644 --- a/elk_metrics_6x/roles/elastic_dependencies/defaults/main.yml +++ b/elk_metrics_6x/roles/elastic_dependencies/defaults/main.yml @@ -15,12 +15,6 @@ # service_group_gid: 5000 # service_owner_uid: 5000 -# Option to define third memory -q_mem: "{{ (ansible_memtotal_mb | int) // 3 }}" - -# Option to define half memory -h_mem: "{{ (ansible_memtotal_mb | int) // 2 }}" - #define this in host/group vars as needed to mount remote filesystems #set the client address as appropriate, eth1 assumes osa container mgmt network #mountpoints and server paths are just examples diff --git a/elk_metrics_6x/roles/elastic_dependencies/vars/vars_elasticsearch.yml b/elk_metrics_6x/roles/elastic_dependencies/vars/vars_elasticsearch.yml index 25e04828..e82f639c 100644 --- a/elk_metrics_6x/roles/elastic_dependencies/vars/vars_elasticsearch.yml +++ b/elk_metrics_6x/roles/elastic_dependencies/vars/vars_elasticsearch.yml @@ -14,4 +14,4 @@ # The heap size is set using the a half of the total memory available with # a cap of 32GiB. If the total available memory is less than 32GiB a buffer of # 10% will be used to ensure the underlying system is not starved of memory. -_elastic_heap_size_default: "{{ ((h_mem | int) > 30720) | ternary(30720, ((h_mem | int) - ((h_mem | int) * 0.1))) }}" +_elastic_heap_size_default: "{{ ((elastic_memory_upper_limit | int) > 30720) | ternary(30720, ((elastic_memory_upper_limit | int) - ((elastic_memory_upper_limit | int) * 0.1))) }}" diff --git a/elk_metrics_6x/roles/elastic_dependencies/vars/vars_logstash.yml b/elk_metrics_6x/roles/elastic_dependencies/vars/vars_logstash.yml index bfb734ff..a5a0c4bd 100644 --- a/elk_metrics_6x/roles/elastic_dependencies/vars/vars_logstash.yml +++ b/elk_metrics_6x/roles/elastic_dependencies/vars/vars_logstash.yml @@ -14,4 +14,4 @@ # The heap size is set using the a quarter of the total memory available with # a cap of 32GiB. If the total available memory is less than 32GiB a buffer of # 10% will be used to ensure the underlying system is not starved of memory. -_elastic_heap_size_default: "{{ ((q_mem | int) > 30720) | ternary(30720, ((q_mem | int) - ((q_mem | int) * 0.1))) }}" +_elastic_heap_size_default: "{{ ((elastic_memory_lower_limit | int) > 30720) | ternary(30720, ((elastic_memory_lower_limit | int) - ((elastic_memory_lower_limit | int) * 0.1))) }}" diff --git a/elk_metrics_6x/roles/elastic_logstash/tasks/main.yml b/elk_metrics_6x/roles/elastic_logstash/tasks/main.yml index 7fc258ff..2e733149 100644 --- a/elk_metrics_6x/roles/elastic_logstash/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_logstash/tasks/main.yml @@ -160,7 +160,7 @@ - what: "tmpfs" where: "{{ logstash_queue_path.stdout.strip() }}" type: "tmpfs" - options: "size={{ (q_mem | int) // 2 }}m,uid={{ logstash_uid.stdout }},gid={{ logstash_gid.stdout }},nodev,nodiratime,noatime" + options: "size={{ (elastic_memory_lower_limit | int) // 2 }}m,uid={{ logstash_uid.stdout }},gid={{ logstash_gid.stdout }},nodev,nodiratime,noatime" unit: Before: - logstash.service @@ -174,7 +174,7 @@ path: "{{ logstash_queue_path.stdout.strip() }}" src: tmpfs fstype: tmpfs - opts: size={{ (q_mem | int) // 2 }}m + opts: size={{ (elastic_memory_lower_limit | int) // 2 }}m state: mounted when: - ansible_service_mgr != 'systemd' diff --git a/elk_metrics_6x/roles/elastic_logstash/templates/logstash.yml.j2 b/elk_metrics_6x/roles/elastic_logstash/templates/logstash.yml.j2 index dbcefd51..f4d0559e 100644 --- a/elk_metrics_6x/roles/elastic_logstash/templates/logstash.yml.j2 +++ b/elk_metrics_6x/roles/elastic_logstash/templates/logstash.yml.j2 @@ -159,7 +159,7 @@ queue.type: persisted {% if logstash_queue_type == 'memory' %} # An in memory queue is being used. The actual size of the queue is 90% of the # total memory limit, which is set using 50% of the heap size. -{% set _memory_queue_size = ((q_mem | int) // 2) %} +{% set _memory_queue_size = ((elastic_memory_lower_limit | int) // 2) %} {% set _memory_queue_size_buffer = (((_memory_queue_size | int) * 0.1) | int) %} queue.max_bytes: {{ (_memory_queue_size | int) - (_memory_queue_size_buffer | int) }}mb {% else %} diff --git a/elk_metrics_6x/tests/test-vars.yml b/elk_metrics_6x/tests/test-vars.yml index b7f5a2cc..b86cac2b 100644 --- a/elk_metrics_6x/tests/test-vars.yml +++ b/elk_metrics_6x/tests/test-vars.yml @@ -18,7 +18,7 @@ physical_host: localhost # NOTE(cloudnull): Test configs used to minimize the impact of a # multi-node install with limited resources. q_storage: 1 -q_mem: 512 -h_mem: 512 +elastic_memory_lower_limit: 512 +elastic_memory_upper_limit: 512 osa_test_repo: "openstack/openstack-ansible-ops" diff --git a/elk_metrics_6x/vars/variables.yml b/elk_metrics_6x/vars/variables.yml index 17cb80f4..9f6fc36d 100644 --- a/elk_metrics_6x/vars/variables.yml +++ b/elk_metrics_6x/vars/variables.yml @@ -1,9 +1,6 @@ --- -# Option to define quarter memory -q_mem: "{{ (ansible_memtotal_mb | int) // 4 }}" - -# Option to define half memory -h_mem: "{{ (ansible_memtotal_mb | int) // 2 }}" +elastic_memory_lower_limit: "{{ (ansible_memtotal_mb | int) * 0.15 // 1 }}" +elastic_memory_upper_limit: "{{ (ansible_memtotal_mb | int) * 0.35 // 1 }}" # Option to set persistent queue storage in gigabytes q_storage: "{{ (ansible_processor_count | int) * (ansible_processor_threads_per_core | int) * 2 }}"